Merge pull request #1458 from severi/main

Improve error logging with a stacktrace
stanfordnlp · Sep 21, 2024 · c617ff8 · c617ff8
2 parents 294d4fd + e0c7b37
commit c617ff8
Showing 1 changed file with 9 additions and 4 deletions.
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.py
@@ -4,6 +4,7 @@
 import threading
 import types
 from typing import Any
+import traceback
 
 import pandas as pd
 import tqdm
@@ -53,6 +54,7 @@ def __init__(
         max_errors=5,
         return_all_scores=False,
         return_outputs=False,
+        provide_traceback=False,
         **_kwargs,
     ):
         self.devset = devset
@@ -66,7 +68,7 @@ def __init__(
         self.cancel_jobs = threading.Event()
         self.return_all_scores = return_all_scores
         self.return_outputs = return_outputs
-
+        self.provide_traceback = provide_traceback
         if "display" in _kwargs:
             dspy.logger.warning(
                 "DeprecationWarning: 'display' has been deprecated. To see all information for debugging,"
@@ -194,9 +196,12 @@ def wrapped_program(example_idx, example):
                     current_error_count = self.error_count
                 if current_error_count >= self.max_errors:
                     raise e
-
-                dspy.logger.error(f"Error for example in dev set: \t\t {e}")
-
+
+                if self.provide_traceback:
+                    dspy.logger.error(f"Error for example in dev set: \t\t {e}\n\twith inputs:\n\t\t{example.inputs()}\n\nStack trace:\n\t{traceback.format_exc()}")
+                else:
+                    dspy.logger.error(f"Error for example in dev set: \t\t {e}. Set `provide_traceback=True` to see the stack trace.")
+
                 return example_idx, example, {}, 0.0
             finally:
                 if creating_new_thread: