Fix no response / null response from LLM

src/crewai/agents/agent_builder/base_agent_executor_mixin.py

@@ -8,6 +8,7 @@
 from crewai.utilities.converter import ConverterError
 from crewai.utilities.evaluators.task_evaluator import TaskEvaluator
 from crewai.utilities.printer import Printer
+from crewai.utilities.types import LLMMessage
 
 if TYPE_CHECKING:
     from crewai.agents.agent_builder.base_agent import BaseAgent
@@ -21,7 +22,7 @@ class CrewAgentExecutorMixin:
     task: "Task"
     iterations: int
     max_iter: int
-    messages: list[dict[str, str]]
+    messages: list[LLMMessage]
     _i18n: I18N
     _printer: Printer = Printer()
 

src/crewai/agents/crew_agent_executor.py

@@ -5,7 +5,7 @@
 """
 
 from collections.abc import Callable
-from typing import Any
+from typing import Any, Literal
 
 from crewai.agents.agent_builder.base_agent import BaseAgent
 from crewai.agents.agent_builder.base_agent_executor_mixin import CrewAgentExecutorMixin
@@ -40,6 +40,7 @@
 from crewai.utilities.constants import TRAINING_DATA_FILE
 from crewai.utilities.tool_utils import execute_tool_and_check_finality
 from crewai.utilities.training_handler import CrewTrainingHandler
+from crewai.utilities.types import LLMMessage
 
 
 class CrewAgentExecutor(CrewAgentExecutorMixin):
@@ -111,7 +112,7 @@ def __init__(
         self.respect_context_window = respect_context_window
         self.request_within_rpm_limit = request_within_rpm_limit
         self.ask_for_human_input = False
-        self.messages: list[dict[str, str]] = []
+        self.messages: list[LLMMessage] = []
         self.iterations = 0
         self.log_error_after = 3
         existing_stop = self.llm.stop or []
@@ -241,7 +242,7 @@ def _invoke_loop(self) -> AgentFinish:
                 if e.__class__.__module__.startswith("litellm"):
                     # Do not retry on litellm errors
                     raise e
-                if is_context_length_exceeded(e):
+                if is_context_length_exceeded(exception=e, messages=self.messages, llm=self.llm):
                     handle_context_length(
                         respect_context_window=self.respect_context_window,
                         printer=self._printer,
@@ -309,7 +310,7 @@ def _invoke_step_callback(
         if self.step_callback:
             self.step_callback(formatted_answer)
 
-    def _append_message(self, text: str, role: str = "assistant") -> None:
+    def _append_message(self, text: str, role: Literal["user", "assistant", "system"] = "assistant") -> None:
         """Add message to conversation history.
 
         Args:

src/crewai/utilities/agent_utils.py

@@ -287,7 +287,7 @@ def process_llm_response(
 def handle_agent_action_core(
     formatted_answer: AgentAction,
     tool_result: ToolResult,
-    messages: list[dict[str, str]] | None = None,
+    messages: list[LLMMessage] | None = None,
     step_callback: Callable | None = None,
     show_logs: Callable | None = None,
 ) -> AgentAction | AgentFinish:
@@ -384,18 +384,54 @@ def handle_output_parser_exception(
     return formatted_answer
 
 
-def is_context_length_exceeded(exception: Exception) -> bool:
-    """Check if the exception is due to context length exceeding.
+def is_context_length_exceeded(
+    exception: Exception,
+    messages: list[LLMMessage],
+    llm: LLM | BaseLLM,
+) -> bool:
+    """
+    Check if the exception is due to context length exceeding or
+    response is empty because context length exceeded.
 
     Args:
         exception: The exception to check
+        messages: Messages sent to the LLM
+        llm: The LLM instance
 
     Returns:
-        bool: True if the exception is due to context length exceeding
+        True if the exception is due to context length exceeding or
+        the response is empty because of context length.
     """
-    return LLMContextLengthExceededError(str(exception))._is_context_limit_error(
+    exceeded_error = LLMContextLengthExceededError(str(exception))._is_context_limit_error(
         str(exception)
     )
+    null_response = is_null_response_because_context_length_exceeded(
+        exception=exception, messages=messages, llm=llm
+    )
+    return exceeded_error or null_response
+
+
+def is_null_response_because_context_length_exceeded(
+    exception: Exception,
+    messages: list[LLMMessage],
+    llm: LLM | BaseLLM,
+) -> bool:
+    """Check if the response is null/empty because context length excedded.
+
+    Args:
+        exception: The exception to check
+
+    Returns:
+        bool: True if the exception is due to context length exceeding
+    """
+    messages_string = " ".join([message["content"] for message in messages])
+    cut_size = llm.get_context_window_size()
+
+    messages_groups = [
+        {"content": messages_string[i : i + cut_size]}
+        for i in range(0, len(messages_string), cut_size)
+    ]
+    return ((len(messages_groups) > 1) and isinstance(exception, ValueError) and "None or empty" in str(exception))
 
 
 def handle_context_length(

src/crewai/utilities/evaluators/task_evaluator.py

@@ -4,6 +4,7 @@
 
 from pydantic import BaseModel, Field
 
+from crewai.agents.agent_builder.base_agent import BaseAgent
 from crewai.events.event_bus import crewai_event_bus
 from crewai.events.types.task_events import TaskEvaluationEvent
 from crewai.llm import LLM
@@ -12,7 +13,6 @@
 from crewai.utilities.training_converter import TrainingConverter
 
 if TYPE_CHECKING:
-    from crewai.agent import Agent
     from crewai.task import Task
 
 
@@ -55,7 +55,7 @@ class TaskEvaluator:
         original_agent: The agent to evaluate.
     """
 
-    def __init__(self, original_agent: Agent) -> None:
+    def __init__(self, original_agent: BaseAgent) -> None:
         """Initializes the TaskEvaluator with the given LLM and agent.
 
         Args:

src/crewai/utilities/tool_utils.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING
 
+from crewai.agents.agent_builder.base_agent import BaseAgent
 from crewai.agents.parser import AgentAction
 from crewai.agents.tools_handler import ToolsHandler
 from crewai.security.fingerprint import Fingerprint

tests/agents/test_agent.py

@@ -1541,6 +1541,55 @@ def test_handle_context_length_exceeds_limit():
         mock_summarize.assert_called_once()
 
 
+def test_handle_context_length_exceeds_no_response():
+    mock_agent_finish = MagicMock(spec=AgentFinish)
+    mock_agent_finish.output = "This is the final answer"
+
+    llm = LLM(model="gpt-4o-mini",context_window_size = 2)
+    llm.context_window_size = 2 # Manually overriding it to be 2 
+
+    exception_to_be_raised =  ValueError("Invalid response from LLM call - None or empty.")
+
+    with patch("crewai.agents.crew_agent_executor.handle_max_iterations_exceeded", return_value = mock_agent_finish) as mock_handle_max_iterations_exceeded:
+        with patch("crewai.agents.crew_agent_executor.get_llm_response") as mock_get_llm_response:
+            with patch("crewai.utilities.agent_utils.is_null_response_because_context_length_exceeded",return_value = True) as mock_is_null_response_because_context_length_exceeded:
+                with patch("crewai.utilities.agent_utils.summarize_messages") as mock_summarize_messages:
+                    mock_get_llm_response.side_effect = exception_to_be_raised
+
+                    agent = Agent(
+                        role="test role",
+                        goal="test goal",
+                        backstory="test backstory",
+                        respect_context_window=True,
+                        llm=llm,
+                        max_iter = 0
+                    )
+
+                    task = Task(description="The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.",expected_output="The final answer")
+                    result = agent.execute_task(
+                        task=task,
+                    )
+
+                    mock_get_llm_response.assert_called_once()
+                    mock_is_null_response_because_context_length_exceeded.assert_called_once_with(
+                        exception = exception_to_be_raised,
+                        messages=[{'role': 'system', 'content': 'You are test role. test backstory\nYour personal goal is: test goal\nTo give my best complete final answer to the task respond using the exact following format:\n\nThought: I now can give a great answer\nFinal Answer: Your final answer must be the great and the most complete as possible, it must be outcome described.\n\nI MUST use these formats, my job depends on it!'}, {'role': 'user', 'content': "\nCurrent Task: The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.\n\nThis is the expected criteria for your final answer: The final answer\nyou MUST return the actual complete content as the final answer, not a summary.\n\nBegin! This is VERY important to you, use the tools available and give your best Final Answer, your job depends on it!\n\nThought:"}],
+                        llm = llm
+                    )
+
+                    mock_summarize_messages.assert_called_once()
+
+                    assert mock_is_null_response_because_context_length_exceeded(
+                        exception = exception_to_be_raised,
+                        messages=[{'role': 'system', 'content': 'You are test role. test backstory\nYour personal goal is: test goal\nTo give my best complete final answer to the task respond using the exact following format:\n\nThought: I now can give a great answer\nFinal Answer: Your final answer must be the great and the most complete as possible, it must be outcome described.\n\nI MUST use these formats, my job depends on it!'}, {'role': 'user', 'content': "\nCurrent Task: The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.\n\nThis is the expected criteria for your final answer: The final answer\nyou MUST return the actual complete content as the final answer, not a summary.\n\nBegin! This is VERY important to you, use the tools available and give your best Final Answer, your job depends on it!\n\nThought:"}],
+                        llm = llm
+                    ) is True
+
+                    mock_summarize_messages.assert_called_once()
+                    mock_handle_max_iterations_exceeded.assert_called_once()
+                    assert result == "This is the final answer"
+
+
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_handle_context_length_exceeds_limit_cli_no():
     agent = Agent(

tests/utilities/test_agent_utils.py

@@ -0,0 +1,92 @@
+import pytest
+from unittest.mock import MagicMock
+from crewai.utilities.agent_utils import is_null_response_because_context_length_exceeded
+
+def test_is_null_response_because_context_length_exceeded_true():
+    """
+    Test that the function returns True when the exception is a ValueError
+    with 'None or empty' and there are messages.
+    """
+    # Arrange
+    mock_llm = MagicMock()
+    mock_llm.get_context_window_size.return_value = 10
+    exception = ValueError("Invalid response from LLM call - None or empty.")
+    messages = [{"content": "This is a test message."}]
+
+    # Act
+    result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm)
+
+    # Assert
+    assert result is True
+
+
+def test_is_null_response_because_context_length_exceeded_false_wrong_exception():
+    """
+    Test that the function returns False when the exception is not a ValueError.
+    """
+    # Arrange
+    mock_llm = MagicMock()
+    mock_llm.get_context_window_size.return_value = 10
+    exception = TypeError("Some other error.")
+    messages = [{"content": "This is a test message."}]
+
+    # Act
+    result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm)
+
+    # Assert
+    assert result is False
+
+
+def test_is_null_response_because_context_length_exceeded_false_wrong_message():
+    """
+    Test that the function returns False when the exception message does not
+    contain 'None or empty'.
+    """
+    # Arrange
+    mock_llm = MagicMock()
+    mock_llm.get_context_window_size.return_value = 10
+    exception = ValueError("Another value error.")
+    messages = [{"content": "This is a test message."}]
+
+    # Act
+    result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm)
+
+    # Assert
+    assert result is False
+
+
+def test_is_null_response_because_context_length_exceeded_false_empty_messages():
+    """
+    Test that the function returns False when the messages list is empty.
+    """
+    # Arrange
+    mock_llm = MagicMock()
+    mock_llm.get_context_window_size.return_value = 10
+    exception = ValueError("Invalid response from LLM call - None or empty.")
+    messages = []
+
+    # Act
+    result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm)
+
+    # Assert
+    assert result is False
+
+
+def test_is_null_response_because_context_length_exceeded_false():
+    """
+    Test that the function returns True when the exception is a ValueError
+    with 'None or empty' and there are messages.
+    """
+    # Arrange
+    mock_llm = MagicMock()
+    mock_llm.get_context_window_size.return_value = 50
+    exception = ValueError("Invalid response from LLM call - None or empty.")
+    messages = [{"content": "This is a test message."}]
+
+    # Act
+    result = is_null_response_because_context_length_exceeded(exception, messages, mock_llm)
+
+    # Assert
+    assert result is False
+
+