Bug Fixes

Author: Siba Rajendran

src/fmcore/llm/mixins/llm_mixins.py

@@ -15,4 +15,4 @@ class LLMConfigMixin(MutableTyped, Mixin):
         llm_config (Optional[LLMConfig]): The LLM configuration object.
     """
 
-    llm_config: Union[LLMConfig, DistributedLLMConfig] = Field(union_mode='left_to_right')
+    llm_config: Union[LLMConfig, DistributedLLMConfig] = Field(union_mode="left_to_right")

src/fmcore/prompt_tuner/evaluator/llm_as_a_judge_boolean/llm_as_a_judge_boolean_evaluator.py

@@ -73,35 +73,41 @@ def _get_instance(cls, *, evaluator_config: EvaluatorConfig) -> "LLMAsJudgeBoole
 
     def evaluate(self, data: Dict) -> bool:
         """
-        Processes the input data by using the llm_as_a_judge_boolean_mapper to evaluate the context.
+        Processes the input data using the llm_as_a_judge_boolean_mapper to evaluate the context.
 
         Args:
             data (BooleanLLMJudgeInput): Input data containing context for evaluation.
 
         Returns:
-            BooleanLLMJudgeOutput: Evaluation result as a boolean decision.
+            bool: Evaluation result as a boolean decision.
         """
-        formatted_message = None
-        llm_response = None
-        json_response = None
-        decision = None
+        formatted_message = llm_response = json_response = decision = None
 
         try:
             formatted_message = self.text_prompt_mapper.map(data)
             llm_response = self.llm_inference_mapper.map([formatted_message])
             json_response = self.json_mapper.map(llm_response.content)
             decision = self.criteria_checker.map(json_response)
-            return decision
+
+            if not isinstance(decision, bool):
+                raise ValueError("Decision is not a boolean value")
+
         except Exception as e:
             Log.error(
-                "Exception during aevaluate:\n"
-                f"formatted_message: {formatted_message}\n"
-                f"llm_response: {llm_response}\n"
-                f"json_response: {json_response}\n"
-                f"decision: {decision}"
+                "[SYNC EVALUATION ERROR]\n"
+                "------------------------------\n"
+                f"[INPUT DATA]: {data}\n\n"
+                f"[PROMPT]: {self.evaluator_config.evaluator_params.prompt}\n\n"
+                f"[FORMATTED MESSAGE]: {formatted_message}\n\n"
+                f"[LLM RESPONSE]: {llm_response}\n\n"
+                f"[JSON RESPONSE]: {json_response}\n\n"
+                f"[DECISION]: {decision}\n\n"
+                f"[ERROR]: {e}"
             )
             raise
 
+        return decision
+
     async def aevaluate(self, data: Dict) -> bool:
         """
         Asynchronous version of `evaluate` that processes the input data.
@@ -110,29 +116,31 @@ async def aevaluate(self, data: Dict) -> bool:
             data (BooleanLLMJudgeInput): Input data containing context for evaluation.
 
         Returns:
-            BooleanLLMJudgeOutput: Evaluation result as a boolean decision.
+            bool: Evaluation result as a boolean decision.
         """
-
-        formatted_message = None
-        llm_response = None
-        json_response = None
-        decision = None
+        formatted_message = llm_response = json_response = decision = None
 
         try:
             formatted_message = await self.text_prompt_mapper.amap(data)
             llm_response = await self.llm_inference_mapper.amap([formatted_message])
             json_response = await self.json_mapper.amap(llm_response.content)
             decision = await self.criteria_checker.amap(json_response)
+
             if not isinstance(decision, bool):
                 raise ValueError("Decision is not a boolean value")
+
         except Exception as e:
             Log.error(
-                "Exception during aevaluate:\n"
-                f"formatted_message: {formatted_message}\n"
-                f"llm_response: {llm_response}\n"
-                f"json_response: {json_response}\n"
-                f"decision: {decision}"
+                "[ASYNC EVALUATION ERROR]\n"
+                "--------------------------------\n"
+                f"[INPUT DATA]: {data}\n\n"
+                f"[PROMPT]: {self.evaluator_config.evaluator_params.prompt}\n\n"
+                f"[FORMATTED MESSAGE]: {formatted_message}\n\n"
+                f"[LLM RESPONSE]: {llm_response}\n\n"
+                f"[JSON RESPONSE]: {json_response}\n\n"
+                f"[DECISION]: {decision}\n\n"
+                f"[ERROR]: {e}"
             )
-            raise e
+            raise
 
         return decision

src/fmcore/prompt_tuner/types/mixins/optimizer_mixins.py

@@ -16,7 +16,7 @@ class StudentConfigMixin(MutableTyped, Mixin):
         student_config (Optional[LLMConfig]): The LLM configuration object for student model
     """
 
-    student_config: Union[LLMConfig, DistributedLLMConfig] = Field(union_mode='left_to_right')
+    student_config: Union[LLMConfig, DistributedLLMConfig] = Field(union_mode="left_to_right")
 
 
 class TeacherConfigMixin(MutableTyped, Mixin):
@@ -27,7 +27,7 @@ class TeacherConfigMixin(MutableTyped, Mixin):
         teacher_config (Optional[LLMConfig]): The LLM configuration object for teacher model
     """
 
-    teacher_config: Union[LLMConfig, DistributedLLMConfig] = Field(union_mode='left_to_right')
+    teacher_config: Union[LLMConfig, DistributedLLMConfig] = Field(union_mode="left_to_right")
 
 
 class EvaluatorConfigMixin(MutableTyped, Mixin):