pydantic_evals.reporting

ReportCase dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

A single case in an evaluation report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

@dataclass(kw_only=True) class ReportCase(Generic[InputsT, OutputT, MetadataT]):  """A single case in an evaluation report.""" name: str  """The name of the [case][pydantic_evals.Case].""" inputs: InputsT  """The inputs to the task, from [`Case.inputs`][pydantic_evals.Case.inputs].""" metadata: MetadataT | None  """Any metadata associated with the case, from [`Case.metadata`][pydantic_evals.Case.metadata].""" expected_output: OutputT | None  """The expected output of the task, from [`Case.expected_output`][pydantic_evals.Case.expected_output].""" output: OutputT  """The output of the task execution.""" metrics: dict[str, float | int] attributes: dict[str, Any] scores: dict[str, EvaluationResult[int | float]] labels: dict[str, EvaluationResult[str]] assertions: dict[str, EvaluationResult[bool]] task_duration: float total_duration: float # includes evaluator execution time trace_id: str | None = None  """The trace ID of the case span.""" span_id: str | None = None  """The span ID of the case span.""" evaluator_failures: list[EvaluatorFailure] = field(default_factory=list) 

name instance-attribute

name: str 

The name of the case.

inputs instance-attribute

inputs: InputsT 

The inputs to the task, from Case.inputs.

metadata instance-attribute

metadata: MetadataT | None 

Any metadata associated with the case, from Case.metadata.

expected_output instance-attribute

expected_output: OutputT | None 

The expected output of the task, from Case.expected_output.

output instance-attribute

output: OutputT 

The output of the task execution.

trace_id class-attribute instance-attribute

trace_id: str | None = None 

The trace ID of the case span.

span_id class-attribute instance-attribute

span_id: str | None = None 

The span ID of the case span.

ReportCaseFailure dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

A single case in an evaluation report that failed due to an error during task execution.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

@dataclass(kw_only=True) class ReportCaseFailure(Generic[InputsT, OutputT, MetadataT]):  """A single case in an evaluation report that failed due to an error during task execution.""" name: str  """The name of the [case][pydantic_evals.Case].""" inputs: InputsT  """The inputs to the task, from [`Case.inputs`][pydantic_evals.Case.inputs].""" metadata: MetadataT | None  """Any metadata associated with the case, from [`Case.metadata`][pydantic_evals.Case.metadata].""" expected_output: OutputT | None  """The expected output of the task, from [`Case.expected_output`][pydantic_evals.Case.expected_output].""" error_message: str  """The message of the exception that caused the failure.""" error_stacktrace: str  """The stacktrace of the exception that caused the failure.""" trace_id: str | None = None  """The trace ID of the case span.""" span_id: str | None = None  """The span ID of the case span.""" 

name instance-attribute

name: str 

The name of the case.

inputs instance-attribute

inputs: InputsT 

The inputs to the task, from Case.inputs.

metadata instance-attribute

metadata: MetadataT | None 

Any metadata associated with the case, from Case.metadata.

expected_output instance-attribute

expected_output: OutputT | None 

The expected output of the task, from Case.expected_output.

error_message instance-attribute

error_message: str 

The message of the exception that caused the failure.

error_stacktrace instance-attribute

error_stacktrace: str 

The stacktrace of the exception that caused the failure.

trace_id class-attribute instance-attribute

trace_id: str | None = None 

The trace ID of the case span.

span_id class-attribute instance-attribute

span_id: str | None = None 

The span ID of the case span.

ReportCaseAggregate

Bases: BaseModel

A synthetic case that summarizes a set of cases.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

class ReportCaseAggregate(BaseModel):  """A synthetic case that summarizes a set of cases.""" name: str scores: dict[str, float | int] labels: dict[str, dict[str, float]] metrics: dict[str, float | int] assertions: float | None task_duration: float total_duration: float @staticmethod def average(cases: list[ReportCase]) -> ReportCaseAggregate:  """Produce a synthetic "summary" case by averaging quantitative attributes.""" num_cases = len(cases) if num_cases == 0: return ReportCaseAggregate( name='Averages', scores={}, labels={}, metrics={}, assertions=None, task_duration=0.0, total_duration=0.0, ) def _scores_averages(scores_by_name: list[dict[str, int | float | bool]]) -> dict[str, float]: counts_by_name: dict[str, int] = defaultdict(int) sums_by_name: dict[str, float] = defaultdict(float) for sbn in scores_by_name: for name, score in sbn.items(): counts_by_name[name] += 1 sums_by_name[name] += score return {name: sums_by_name[name] / counts_by_name[name] for name in sums_by_name} def _labels_averages(labels_by_name: list[dict[str, str]]) -> dict[str, dict[str, float]]: counts_by_name: dict[str, int] = defaultdict(int) sums_by_name: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float)) for lbn in labels_by_name: for name, label in lbn.items(): counts_by_name[name] += 1 sums_by_name[name][label] += 1 return { name: {value: count / counts_by_name[name] for value, count in sums_by_name[name].items()} for name in sums_by_name } average_task_duration = sum(case.task_duration for case in cases) / num_cases average_total_duration = sum(case.total_duration for case in cases) / num_cases # average_assertions: dict[str, float] = _scores_averages([{k: v.value for k, v in case.scores.items()} for case in cases]) average_scores: dict[str, float] = _scores_averages( [{k: v.value for k, v in case.scores.items()} for case in cases] ) average_labels: dict[str, dict[str, float]] = _labels_averages( [{k: v.value for k, v in case.labels.items()} for case in cases] ) average_metrics: dict[str, float] = _scores_averages([case.metrics for case in cases]) average_assertions: float | None = None n_assertions = sum(len(case.assertions) for case in cases) if n_assertions > 0: n_passing = sum(1 for case in cases for assertion in case.assertions.values() if assertion.value) average_assertions = n_passing / n_assertions return ReportCaseAggregate( name='Averages', scores=average_scores, labels=average_labels, metrics=average_metrics, assertions=average_assertions, task_duration=average_task_duration, total_duration=average_total_duration, ) 

average staticmethod

average(cases: list[ReportCase]) -> ReportCaseAggregate 

Produce a synthetic "summary" case by averaging quantitative attributes.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

@staticmethod def average(cases: list[ReportCase]) -> ReportCaseAggregate:  """Produce a synthetic "summary" case by averaging quantitative attributes.""" num_cases = len(cases) if num_cases == 0: return ReportCaseAggregate( name='Averages', scores={}, labels={}, metrics={}, assertions=None, task_duration=0.0, total_duration=0.0, ) def _scores_averages(scores_by_name: list[dict[str, int | float | bool]]) -> dict[str, float]: counts_by_name: dict[str, int] = defaultdict(int) sums_by_name: dict[str, float] = defaultdict(float) for sbn in scores_by_name: for name, score in sbn.items(): counts_by_name[name] += 1 sums_by_name[name] += score return {name: sums_by_name[name] / counts_by_name[name] for name in sums_by_name} def _labels_averages(labels_by_name: list[dict[str, str]]) -> dict[str, dict[str, float]]: counts_by_name: dict[str, int] = defaultdict(int) sums_by_name: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float)) for lbn in labels_by_name: for name, label in lbn.items(): counts_by_name[name] += 1 sums_by_name[name][label] += 1 return { name: {value: count / counts_by_name[name] for value, count in sums_by_name[name].items()} for name in sums_by_name } average_task_duration = sum(case.task_duration for case in cases) / num_cases average_total_duration = sum(case.total_duration for case in cases) / num_cases # average_assertions: dict[str, float] = _scores_averages([{k: v.value for k, v in case.scores.items()} for case in cases]) average_scores: dict[str, float] = _scores_averages( [{k: v.value for k, v in case.scores.items()} for case in cases] ) average_labels: dict[str, dict[str, float]] = _labels_averages( [{k: v.value for k, v in case.labels.items()} for case in cases] ) average_metrics: dict[str, float] = _scores_averages([case.metrics for case in cases]) average_assertions: float | None = None n_assertions = sum(len(case.assertions) for case in cases) if n_assertions > 0: n_passing = sum(1 for case in cases for assertion in case.assertions.values() if assertion.value) average_assertions = n_passing / n_assertions return ReportCaseAggregate( name='Averages', scores=average_scores, labels=average_labels, metrics=average_metrics, assertions=average_assertions, task_duration=average_task_duration, total_duration=average_total_duration, ) 

EvaluationReport dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

A report of the results of evaluating a model on a set of cases.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

@dataclass(kw_only=True) class EvaluationReport(Generic[InputsT, OutputT, MetadataT]):  """A report of the results of evaluating a model on a set of cases.""" name: str  """The name of the report.""" cases: list[ReportCase[InputsT, OutputT, MetadataT]]  """The cases in the report.""" failures: list[ReportCaseFailure[InputsT, OutputT, MetadataT]] = field(default_factory=list)  """The failures in the report. These are cases where task execution raised an exception.""" experiment_metadata: dict[str, Any] | None = None  """Metadata associated with the specific experiment represented by this report.""" trace_id: str | None = None  """The trace ID of the evaluation.""" span_id: str | None = None  """The span ID of the evaluation.""" def averages(self) -> ReportCaseAggregate | None: if self.cases: return ReportCaseAggregate.average(self.cases) return None def render( self, width: int | None = None, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_errors: bool = True, include_error_stacktrace: bool = False, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: dict[str, RenderNumberConfig] | None = None, label_configs: dict[str, RenderValueConfig] | None = None, metric_configs: dict[str, RenderNumberConfig] | None = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, ) -> str:  """Render this report to a nicely-formatted string, optionally comparing it to a baseline report.  If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.  """ io_file = StringIO() console = Console(width=width, file=io_file) self.print( width=width, baseline=baseline, console=console, include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=include_output, include_durations=include_durations, include_total_duration=include_total_duration, include_removed_cases=include_removed_cases, include_averages=include_averages, include_errors=include_errors, include_error_stacktrace=include_error_stacktrace, include_evaluator_failures=include_evaluator_failures, input_config=input_config, metadata_config=metadata_config, output_config=output_config, score_configs=score_configs, label_configs=label_configs, metric_configs=metric_configs, duration_config=duration_config, include_reasons=include_reasons, ) return io_file.getvalue() def print( self, width: int | None = None, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None, *, console: Console | None = None, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_errors: bool = True, include_error_stacktrace: bool = False, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: dict[str, RenderNumberConfig] | None = None, label_configs: dict[str, RenderValueConfig] | None = None, metric_configs: dict[str, RenderNumberConfig] | None = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, ) -> None:  """Print this report to the console, optionally comparing it to a baseline report.  If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.  """ if console is None: # pragma: no branch console = Console(width=width) metadata_panel = self._metadata_panel(baseline=baseline) renderable: RenderableType = self.console_table( baseline=baseline, include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=include_output, include_durations=include_durations, include_total_duration=include_total_duration, include_removed_cases=include_removed_cases, include_averages=include_averages, include_evaluator_failures=include_evaluator_failures, input_config=input_config, metadata_config=metadata_config, output_config=output_config, score_configs=score_configs, label_configs=label_configs, metric_configs=metric_configs, duration_config=duration_config, include_reasons=include_reasons, with_title=not metadata_panel, ) # Wrap table with experiment metadata panel if present if metadata_panel: renderable = Group(metadata_panel, renderable) console.print(renderable) if include_errors and self.failures: # pragma: no cover failures_table = self.failures_table( include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_error_message=True, include_error_stacktrace=include_error_stacktrace, input_config=input_config, metadata_config=metadata_config, ) console.print(failures_table, style='red') # TODO(DavidM): in v2, change the return type here to RenderableType def console_table( self, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: dict[str, RenderNumberConfig] | None = None, label_configs: dict[str, RenderValueConfig] | None = None, metric_configs: dict[str, RenderNumberConfig] | None = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, with_title: bool = True, ) -> Table:  """Return a table containing the data from this report.  If a baseline is provided, returns a diff between this report and the baseline report.  Optionally include input and output details.  """ renderer = EvaluationRenderer( include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=include_output, include_durations=include_durations, include_total_duration=include_total_duration, include_removed_cases=include_removed_cases, include_averages=include_averages, include_error_message=False, include_error_stacktrace=False, include_evaluator_failures=include_evaluator_failures, input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})}, metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})}, output_config=output_config or _DEFAULT_VALUE_CONFIG, score_configs=score_configs or {}, label_configs=label_configs or {}, metric_configs=metric_configs or {}, duration_config=duration_config or _DEFAULT_DURATION_CONFIG, include_reasons=include_reasons, ) if baseline is None: return renderer.build_table(self, with_title=with_title) else: return renderer.build_diff_table(self, baseline, with_title=with_title) def _metadata_panel( self, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None ) -> RenderableType | None:  """Wrap a table with an experiment metadata panel if metadata exists.  Args:  table: The table to wrap  baseline: Optional baseline report for diff metadata  Returns:  Either the table unchanged or a Group with Panel and Table  """ if baseline is None: # Single report - show metadata if present if self.experiment_metadata: metadata_text = Text() items = list(self.experiment_metadata.items()) for i, (key, value) in enumerate(items): metadata_text.append(f'{key}: {value}', style='dim') if i < len(items) - 1: metadata_text.append('\n') return Panel( metadata_text, title=f'Evaluation Summary: {self.name}', title_align='left', border_style='dim', padding=(0, 1), expand=False, ) else: # Diff report - show metadata diff if either has metadata if self.experiment_metadata or baseline.experiment_metadata: diff_name = baseline.name if baseline.name == self.name else f'{baseline.name} → {self.name}' metadata_text = Text() lines_styles: list[tuple[str, str]] = [] if baseline.experiment_metadata and self.experiment_metadata: # Collect all keys from both all_keys = sorted(set(baseline.experiment_metadata.keys()) | set(self.experiment_metadata.keys())) for key in all_keys: baseline_val = baseline.experiment_metadata.get(key) report_val = self.experiment_metadata.get(key) if baseline_val == report_val: lines_styles.append((f'{key}: {report_val}', 'dim')) elif baseline_val is None: lines_styles.append((f'+ {key}: {report_val}', 'green')) elif report_val is None: lines_styles.append((f'- {key}: {baseline_val}', 'red')) else: lines_styles.append((f'{key}: {baseline_val} → {report_val}', 'yellow')) elif self.experiment_metadata: lines_styles = [(f'+ {k}: {v}', 'green') for k, v in self.experiment_metadata.items()] else: # baseline.experiment_metadata only assert baseline.experiment_metadata is not None lines_styles = [(f'- {k}: {v}', 'red') for k, v in baseline.experiment_metadata.items()] for i, (line, style) in enumerate(lines_styles): metadata_text.append(line, style=style) if i < len(lines_styles) - 1: metadata_text.append('\n') return Panel( metadata_text, title=f'Evaluation Diff: {diff_name}', title_align='left', border_style='dim', padding=(0, 1), expand=False, ) return None # TODO(DavidM): in v2, change the return type here to RenderableType def failures_table( self, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_error_message: bool = True, include_error_stacktrace: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, ) -> Table:  """Return a table containing the failures in this report.""" renderer = EvaluationRenderer( include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=False, include_durations=False, include_total_duration=False, include_removed_cases=False, include_averages=False, input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})}, metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})}, output_config=_DEFAULT_VALUE_CONFIG, score_configs={}, label_configs={}, metric_configs={}, duration_config=_DEFAULT_DURATION_CONFIG, include_reasons=False, include_error_message=include_error_message, include_error_stacktrace=include_error_stacktrace, include_evaluator_failures=False, # Not applicable for failures table ) return renderer.build_failures_table(self) def __str__(self) -> str: # pragma: lax no cover  """Return a string representation of the report.""" return self.render() 

name instance-attribute

name: str 

The name of the report.

cases instance-attribute

cases: list[ReportCase[InputsT, OutputT, MetadataT]] 

The cases in the report.

failures class-attribute instance-attribute

failures: list[ ReportCaseFailure[InputsT, OutputT, MetadataT] ] = field(default_factory=list) 

The failures in the report. These are cases where task execution raised an exception.

experiment_metadata class-attribute instance-attribute

experiment_metadata: dict[str, Any] | None = None 

Metadata associated with the specific experiment represented by this report.

trace_id class-attribute instance-attribute

trace_id: str | None = None 

The trace ID of the evaluation.

span_id class-attribute instance-attribute

span_id: str | None = None 

The span ID of the evaluation.

render

render( width: int | None = None, baseline: ( EvaluationReport[InputsT, OutputT, MetadataT] | None ) = None, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_errors: bool = True, include_error_stacktrace: bool = False, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: ( dict[str, RenderNumberConfig] | None ) = None, label_configs: ( dict[str, RenderValueConfig] | None ) = None, metric_configs: ( dict[str, RenderNumberConfig] | None ) = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False ) -> str 

Render this report to a nicely-formatted string, optionally comparing it to a baseline report.

If you want more control over the output, use console_table instead and pass it to rich.Console.print.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def render( self, width: int | None = None, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_errors: bool = True, include_error_stacktrace: bool = False, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: dict[str, RenderNumberConfig] | None = None, label_configs: dict[str, RenderValueConfig] | None = None, metric_configs: dict[str, RenderNumberConfig] | None = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, ) -> str:  """Render this report to a nicely-formatted string, optionally comparing it to a baseline report.  If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.  """ io_file = StringIO() console = Console(width=width, file=io_file) self.print( width=width, baseline=baseline, console=console, include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=include_output, include_durations=include_durations, include_total_duration=include_total_duration, include_removed_cases=include_removed_cases, include_averages=include_averages, include_errors=include_errors, include_error_stacktrace=include_error_stacktrace, include_evaluator_failures=include_evaluator_failures, input_config=input_config, metadata_config=metadata_config, output_config=output_config, score_configs=score_configs, label_configs=label_configs, metric_configs=metric_configs, duration_config=duration_config, include_reasons=include_reasons, ) return io_file.getvalue() 

print

print( width: int | None = None, baseline: ( EvaluationReport[InputsT, OutputT, MetadataT] | None ) = None, *, console: Console | None = None, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_errors: bool = True, include_error_stacktrace: bool = False, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: ( dict[str, RenderNumberConfig] | None ) = None, label_configs: ( dict[str, RenderValueConfig] | None ) = None, metric_configs: ( dict[str, RenderNumberConfig] | None ) = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False ) -> None 

Print this report to the console, optionally comparing it to a baseline report.

If you want more control over the output, use console_table instead and pass it to rich.Console.print.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def print( self, width: int | None = None, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None, *, console: Console | None = None, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_errors: bool = True, include_error_stacktrace: bool = False, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: dict[str, RenderNumberConfig] | None = None, label_configs: dict[str, RenderValueConfig] | None = None, metric_configs: dict[str, RenderNumberConfig] | None = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, ) -> None:  """Print this report to the console, optionally comparing it to a baseline report.  If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.  """ if console is None: # pragma: no branch console = Console(width=width) metadata_panel = self._metadata_panel(baseline=baseline) renderable: RenderableType = self.console_table( baseline=baseline, include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=include_output, include_durations=include_durations, include_total_duration=include_total_duration, include_removed_cases=include_removed_cases, include_averages=include_averages, include_evaluator_failures=include_evaluator_failures, input_config=input_config, metadata_config=metadata_config, output_config=output_config, score_configs=score_configs, label_configs=label_configs, metric_configs=metric_configs, duration_config=duration_config, include_reasons=include_reasons, with_title=not metadata_panel, ) # Wrap table with experiment metadata panel if present if metadata_panel: renderable = Group(metadata_panel, renderable) console.print(renderable) if include_errors and self.failures: # pragma: no cover failures_table = self.failures_table( include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_error_message=True, include_error_stacktrace=include_error_stacktrace, input_config=input_config, metadata_config=metadata_config, ) console.print(failures_table, style='red') 

console_table

console_table( baseline: ( EvaluationReport[InputsT, OutputT, MetadataT] | None ) = None, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: ( dict[str, RenderNumberConfig] | None ) = None, label_configs: ( dict[str, RenderValueConfig] | None ) = None, metric_configs: ( dict[str, RenderNumberConfig] | None ) = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, with_title: bool = True ) -> Table 

Return a table containing the data from this report.

If a baseline is provided, returns a diff between this report and the baseline report. Optionally include input and output details.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def console_table( self, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_output: bool = False, include_durations: bool = True, include_total_duration: bool = False, include_removed_cases: bool = False, include_averages: bool = True, include_evaluator_failures: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, output_config: RenderValueConfig | None = None, score_configs: dict[str, RenderNumberConfig] | None = None, label_configs: dict[str, RenderValueConfig] | None = None, metric_configs: dict[str, RenderNumberConfig] | None = None, duration_config: RenderNumberConfig | None = None, include_reasons: bool = False, with_title: bool = True, ) -> Table:  """Return a table containing the data from this report.  If a baseline is provided, returns a diff between this report and the baseline report.  Optionally include input and output details.  """ renderer = EvaluationRenderer( include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=include_output, include_durations=include_durations, include_total_duration=include_total_duration, include_removed_cases=include_removed_cases, include_averages=include_averages, include_error_message=False, include_error_stacktrace=False, include_evaluator_failures=include_evaluator_failures, input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})}, metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})}, output_config=output_config or _DEFAULT_VALUE_CONFIG, score_configs=score_configs or {}, label_configs=label_configs or {}, metric_configs=metric_configs or {}, duration_config=duration_config or _DEFAULT_DURATION_CONFIG, include_reasons=include_reasons, ) if baseline is None: return renderer.build_table(self, with_title=with_title) else: return renderer.build_diff_table(self, baseline, with_title=with_title) 

failures_table

failures_table( *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_error_message: bool = True, include_error_stacktrace: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None ) -> Table 

Return a table containing the failures in this report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def failures_table( self, *, include_input: bool = False, include_metadata: bool = False, include_expected_output: bool = False, include_error_message: bool = True, include_error_stacktrace: bool = True, input_config: RenderValueConfig | None = None, metadata_config: RenderValueConfig | None = None, ) -> Table:  """Return a table containing the failures in this report.""" renderer = EvaluationRenderer( include_input=include_input, include_metadata=include_metadata, include_expected_output=include_expected_output, include_output=False, include_durations=False, include_total_duration=False, include_removed_cases=False, include_averages=False, input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})}, metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})}, output_config=_DEFAULT_VALUE_CONFIG, score_configs={}, label_configs={}, metric_configs={}, duration_config=_DEFAULT_DURATION_CONFIG, include_reasons=False, include_error_message=include_error_message, include_error_stacktrace=include_error_stacktrace, include_evaluator_failures=False, # Not applicable for failures table ) return renderer.build_failures_table(self) 

__str__

__str__() -> str 

Return a string representation of the report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def __str__(self) -> str: # pragma: lax no cover  """Return a string representation of the report.""" return self.render() 

RenderValueConfig

Bases: TypedDict

A configuration for rendering a values in an Evaluation report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

class RenderValueConfig(TypedDict, total=False):  """A configuration for rendering a values in an Evaluation report.""" value_formatter: str | Callable[[Any], str] diff_checker: Callable[[Any, Any], bool] | None diff_formatter: Callable[[Any, Any], str | None] | None diff_style: str 

RenderNumberConfig

Bases: TypedDict

A configuration for rendering a particular score or metric in an Evaluation report.

See the implementation of _RenderNumber for more clarity on how these parameters affect the rendering.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

class RenderNumberConfig(TypedDict, total=False):  """A configuration for rendering a particular score or metric in an Evaluation report.  See the implementation of `_RenderNumber` for more clarity on how these parameters affect the rendering.  """ value_formatter: str | Callable[[float | int], str]  """The logic to use for formatting values.  * If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures.  * You can also use a custom string format spec, e.g. '{:.3f}'  * You can also use a custom function, e.g. lambda x: f'{x:.3f}'  """ diff_formatter: str | Callable[[float | int, float | int], str | None] | None  """The logic to use for formatting details about the diff.  The strings produced by the value_formatter will always be included in the reports, but the diff_formatter is  used to produce additional text about the difference between the old and new values, such as the absolute or  relative difference.  * If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four  significant figures, and will include the percentage change.  * You can also use a custom string format spec, e.g. '{:+.3f}'  * You can also use a custom function, e.g. lambda x: f'{x:+.3f}'.  If this function returns None, no extra diff text will be added.  * You can also use None to never generate extra diff text.  """ diff_atol: float  """The absolute tolerance for considering a difference "significant".  A difference is "significant" if `abs(new - old) < self.diff_atol + self.diff_rtol * abs(old)`.  If a difference is not significant, it will not have the diff styles applied. Note that we still show  both the rendered before and after values in the diff any time they differ, even if the difference is not  significant. (If the rendered values are exactly the same, we only show the value once.)  If not provided, use 1e-6.  """ diff_rtol: float  """The relative tolerance for considering a difference "significant".  See the description of `diff_atol` for more details about what makes a difference "significant".  If not provided, use 0.001 if all values are ints, otherwise 0.05.  """ diff_increase_style: str  """The style to apply to diffed values that have a significant increase.  See the description of `diff_atol` for more details about what makes a difference "significant".  If not provided, use green for scores and red for metrics. You can also use arbitrary `rich` styles, such as "bold red".  """ diff_decrease_style: str  """The style to apply to diffed values that have significant decrease.  See the description of `diff_atol` for more details about what makes a difference "significant".  If not provided, use red for scores and green for metrics. You can also use arbitrary `rich` styles, such as "bold red".  """ 

value_formatter instance-attribute

value_formatter: str | Callable[[float | int], str] 

The logic to use for formatting values.

• If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures.
• You can also use a custom string format spec, e.g. '{:.3f}'
• You can also use a custom function, e.g. lambda x: f'{x:.3f}'

diff_formatter instance-attribute

diff_formatter: ( str | Callable[[float | int, float | int], str | None] | None ) 

The logic to use for formatting details about the diff.

The strings produced by the value_formatter will always be included in the reports, but the diff_formatter is used to produce additional text about the difference between the old and new values, such as the absolute or relative difference.

• If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures, and will include the percentage change.
• You can also use a custom string format spec, e.g. '{:+.3f}'
• You can also use a custom function, e.g. lambda x: f'{x:+.3f}'. If this function returns None, no extra diff text will be added.
• You can also use None to never generate extra diff text.

diff_atol instance-attribute

diff_atol: float 

The absolute tolerance for considering a difference "significant".

A difference is "significant" if abs(new - old) < self.diff_atol + self.diff_rtol * abs(old).

If a difference is not significant, it will not have the diff styles applied. Note that we still show both the rendered before and after values in the diff any time they differ, even if the difference is not significant. (If the rendered values are exactly the same, we only show the value once.)

If not provided, use 1e-6.

diff_rtol instance-attribute

diff_rtol: float 

The relative tolerance for considering a difference "significant".

See the description of diff_atol for more details about what makes a difference "significant".

If not provided, use 0.001 if all values are ints, otherwise 0.05.

diff_increase_style instance-attribute

diff_increase_style: str 

The style to apply to diffed values that have a significant increase.

See the description of diff_atol for more details about what makes a difference "significant".

If not provided, use green for scores and red for metrics. You can also use arbitrary rich styles, such as "bold red".

diff_decrease_style instance-attribute

diff_decrease_style: str 

The style to apply to diffed values that have significant decrease.

See the description of diff_atol for more details about what makes a difference "significant".

If not provided, use red for scores and green for metrics. You can also use arbitrary rich styles, such as "bold red".

EvaluationRenderer dataclass

A class for rendering an EvalReport or the diff between two EvalReports.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

@dataclass(kw_only=True) class EvaluationRenderer:  """A class for rendering an EvalReport or the diff between two EvalReports.""" # Columns to include include_input: bool include_metadata: bool include_expected_output: bool include_output: bool include_durations: bool include_total_duration: bool # Rows to include include_removed_cases: bool include_averages: bool input_config: RenderValueConfig metadata_config: RenderValueConfig output_config: RenderValueConfig score_configs: dict[str, RenderNumberConfig] label_configs: dict[str, RenderValueConfig] metric_configs: dict[str, RenderNumberConfig] duration_config: RenderNumberConfig # Data to include include_reasons: bool # only applies to reports, not to diffs include_error_message: bool include_error_stacktrace: bool include_evaluator_failures: bool def include_scores(self, report: EvaluationReport, baseline: EvaluationReport | None = None): return any(case.scores for case in self._all_cases(report, baseline)) def include_labels(self, report: EvaluationReport, baseline: EvaluationReport | None = None): return any(case.labels for case in self._all_cases(report, baseline)) def include_metrics(self, report: EvaluationReport, baseline: EvaluationReport | None = None): return any(case.metrics for case in self._all_cases(report, baseline)) def include_assertions(self, report: EvaluationReport, baseline: EvaluationReport | None = None): return any(case.assertions for case in self._all_cases(report, baseline)) def include_evaluator_failures_column(self, report: EvaluationReport, baseline: EvaluationReport | None = None): return self.include_evaluator_failures and any( case.evaluator_failures for case in self._all_cases(report, baseline) ) def _all_cases(self, report: EvaluationReport, baseline: EvaluationReport | None) -> list[ReportCase]: if not baseline: return report.cases else: return report.cases + self._baseline_cases_to_include(report, baseline) def _baseline_cases_to_include(self, report: EvaluationReport, baseline: EvaluationReport) -> list[ReportCase]: if self.include_removed_cases: return baseline.cases report_case_names = {case.name for case in report.cases} return [case for case in baseline.cases if case.name in report_case_names] def _get_case_renderer( self, report: EvaluationReport, baseline: EvaluationReport | None = None ) -> ReportCaseRenderer: input_renderer = _ValueRenderer.from_config(self.input_config) metadata_renderer = _ValueRenderer.from_config(self.metadata_config) output_renderer = _ValueRenderer.from_config(self.output_config) score_renderers = self._infer_score_renderers(report, baseline) label_renderers = self._infer_label_renderers(report, baseline) metric_renderers = self._infer_metric_renderers(report, baseline) duration_renderer = _NumberRenderer.infer_from_config( self.duration_config, 'duration', [x.task_duration for x in self._all_cases(report, baseline)] ) return ReportCaseRenderer( include_input=self.include_input, include_metadata=self.include_metadata, include_expected_output=self.include_expected_output, include_output=self.include_output, include_scores=self.include_scores(report, baseline), include_labels=self.include_labels(report, baseline), include_metrics=self.include_metrics(report, baseline), include_assertions=self.include_assertions(report, baseline), include_reasons=self.include_reasons, include_durations=self.include_durations, include_total_duration=self.include_total_duration, include_error_message=self.include_error_message, include_error_stacktrace=self.include_error_stacktrace, include_evaluator_failures=self.include_evaluator_failures_column(report, baseline), input_renderer=input_renderer, metadata_renderer=metadata_renderer, output_renderer=output_renderer, score_renderers=score_renderers, label_renderers=label_renderers, metric_renderers=metric_renderers, duration_renderer=duration_renderer, ) # TODO(DavidM): in v2, change the return type here to RenderableType def build_table(self, report: EvaluationReport, *, with_title: bool = True) -> Table:  """Build a table for the report.  Args:  report: The evaluation report to render  with_title: Whether to include the title in the table (default True)  Returns:  A Rich Table object  """ case_renderer = self._get_case_renderer(report) title = f'Evaluation Summary: {report.name}' if with_title else '' table = case_renderer.build_base_table(title) for case in report.cases: table.add_row(*case_renderer.build_row(case)) if self.include_averages: # pragma: no branch average = report.averages() if average: # pragma: no branch table.add_row(*case_renderer.build_aggregate_row(average)) return table # TODO(DavidM): in v2, change the return type here to RenderableType def build_diff_table( self, report: EvaluationReport, baseline: EvaluationReport, *, with_title: bool = True ) -> Table:  """Build a diff table comparing report to baseline.  Args:  report: The evaluation report to compare  baseline: The baseline report to compare against  with_title: Whether to include the title in the table (default True)  Returns:  A Rich Table object  """ report_cases = report.cases baseline_cases = self._baseline_cases_to_include(report, baseline) report_cases_by_id = {case.name: case for case in report_cases} baseline_cases_by_id = {case.name: case for case in baseline_cases} diff_cases: list[tuple[ReportCase, ReportCase]] = [] removed_cases: list[ReportCase] = [] added_cases: list[ReportCase] = [] for case_id in sorted(set(baseline_cases_by_id.keys()) | set(report_cases_by_id.keys())): maybe_baseline_case = baseline_cases_by_id.get(case_id) maybe_report_case = report_cases_by_id.get(case_id) if maybe_baseline_case and maybe_report_case: diff_cases.append((maybe_baseline_case, maybe_report_case)) elif maybe_baseline_case: removed_cases.append(maybe_baseline_case) elif maybe_report_case: added_cases.append(maybe_report_case) else: # pragma: no cover assert False, 'This should be unreachable' case_renderer = self._get_case_renderer(report, baseline) diff_name = baseline.name if baseline.name == report.name else f'{baseline.name} → {report.name}' title = f'Evaluation Diff: {diff_name}' if with_title else '' table = case_renderer.build_base_table(title) for baseline_case, new_case in diff_cases: table.add_row(*case_renderer.build_diff_row(new_case, baseline_case)) for case in added_cases: row = case_renderer.build_row(case) row[0] = f'[green]+ Added Case[/]\n{row[0]}' table.add_row(*row) for case in removed_cases: row = case_renderer.build_row(case) row[0] = f'[red]- Removed Case[/]\n{row[0]}' table.add_row(*row) if self.include_averages: # pragma: no branch report_average = ReportCaseAggregate.average(report_cases) baseline_average = ReportCaseAggregate.average(baseline_cases) table.add_row(*case_renderer.build_diff_aggregate_row(report_average, baseline_average)) return table # TODO(DavidM): in v2, change the return type here to RenderableType def build_failures_table(self, report: EvaluationReport) -> Table: case_renderer = self._get_case_renderer(report) table = case_renderer.build_failures_table('Case Failures') for case in report.failures: table.add_row(*case_renderer.build_failure_row(case)) return table def _infer_score_renderers( self, report: EvaluationReport, baseline: EvaluationReport | None ) -> dict[str, _NumberRenderer]: all_cases = self._all_cases(report, baseline) values_by_name: dict[str, list[float | int]] = {} for case in all_cases: for k, score in case.scores.items(): values_by_name.setdefault(k, []).append(score.value) all_renderers: dict[str, _NumberRenderer] = {} for name, values in values_by_name.items(): merged_config = _DEFAULT_NUMBER_CONFIG.copy() merged_config.update(self.score_configs.get(name, {})) all_renderers[name] = _NumberRenderer.infer_from_config(merged_config, 'score', values) return all_renderers def _infer_label_renderers( self, report: EvaluationReport, baseline: EvaluationReport | None ) -> dict[str, _ValueRenderer]: all_cases = self._all_cases(report, baseline) all_names: set[str] = set() for case in all_cases: for k in case.labels: all_names.add(k) all_renderers: dict[str, _ValueRenderer] = {} for name in all_names: merged_config = _DEFAULT_VALUE_CONFIG.copy() merged_config.update(self.label_configs.get(name, {})) all_renderers[name] = _ValueRenderer.from_config(merged_config) return all_renderers def _infer_metric_renderers( self, report: EvaluationReport, baseline: EvaluationReport | None ) -> dict[str, _NumberRenderer]: all_cases = self._all_cases(report, baseline) values_by_name: dict[str, list[float | int]] = {} for case in all_cases: for k, v in case.metrics.items(): values_by_name.setdefault(k, []).append(v) all_renderers: dict[str, _NumberRenderer] = {} for name, values in values_by_name.items(): merged_config = _DEFAULT_NUMBER_CONFIG.copy() merged_config.update(self.metric_configs.get(name, {})) all_renderers[name] = _NumberRenderer.infer_from_config(merged_config, 'metric', values) return all_renderers def _infer_duration_renderer( self, report: EvaluationReport, baseline: EvaluationReport | None ) -> _NumberRenderer: # pragma: no cover all_cases = self._all_cases(report, baseline) all_durations = [x.task_duration for x in all_cases] if self.include_total_duration: all_durations += [x.total_duration for x in all_cases] return _NumberRenderer.infer_from_config(self.duration_config, 'duration', all_durations) 

build_table

build_table( report: EvaluationReport, *, with_title: bool = True ) -> Table 

Build a table for the report.

Parameters:

Name	Type	Description	Default
report	EvaluationReport	The evaluation report to render	required
with_title	bool	Whether to include the title in the table (default True)	True

Returns:

Type	Description
Table	A Rich Table object

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def build_table(self, report: EvaluationReport, *, with_title: bool = True) -> Table:  """Build a table for the report.  Args:  report: The evaluation report to render  with_title: Whether to include the title in the table (default True)  Returns:  A Rich Table object  """ case_renderer = self._get_case_renderer(report) title = f'Evaluation Summary: {report.name}' if with_title else '' table = case_renderer.build_base_table(title) for case in report.cases: table.add_row(*case_renderer.build_row(case)) if self.include_averages: # pragma: no branch average = report.averages() if average: # pragma: no branch table.add_row(*case_renderer.build_aggregate_row(average)) return table 

build_diff_table

build_diff_table( report: EvaluationReport, baseline: EvaluationReport, *, with_title: bool = True ) -> Table 

Build a diff table comparing report to baseline.

Parameters:

Name	Type	Description	Default
report	EvaluationReport	The evaluation report to compare	required
baseline	EvaluationReport	The baseline report to compare against	required
with_title	bool	Whether to include the title in the table (default True)	True

Returns:

Type	Description
Table	A Rich Table object

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py

def build_diff_table( self, report: EvaluationReport, baseline: EvaluationReport, *, with_title: bool = True ) -> Table:  """Build a diff table comparing report to baseline.  Args:  report: The evaluation report to compare  baseline: The baseline report to compare against  with_title: Whether to include the title in the table (default True)  Returns:  A Rich Table object  """ report_cases = report.cases baseline_cases = self._baseline_cases_to_include(report, baseline) report_cases_by_id = {case.name: case for case in report_cases} baseline_cases_by_id = {case.name: case for case in baseline_cases} diff_cases: list[tuple[ReportCase, ReportCase]] = [] removed_cases: list[ReportCase] = [] added_cases: list[ReportCase] = [] for case_id in sorted(set(baseline_cases_by_id.keys()) | set(report_cases_by_id.keys())): maybe_baseline_case = baseline_cases_by_id.get(case_id) maybe_report_case = report_cases_by_id.get(case_id) if maybe_baseline_case and maybe_report_case: diff_cases.append((maybe_baseline_case, maybe_report_case)) elif maybe_baseline_case: removed_cases.append(maybe_baseline_case) elif maybe_report_case: added_cases.append(maybe_report_case) else: # pragma: no cover assert False, 'This should be unreachable' case_renderer = self._get_case_renderer(report, baseline) diff_name = baseline.name if baseline.name == report.name else f'{baseline.name} → {report.name}' title = f'Evaluation Diff: {diff_name}' if with_title else '' table = case_renderer.build_base_table(title) for baseline_case, new_case in diff_cases: table.add_row(*case_renderer.build_diff_row(new_case, baseline_case)) for case in added_cases: row = case_renderer.build_row(case) row[0] = f'[green]+ Added Case[/]\n{row[0]}' table.add_row(*row) for case in removed_cases: row = case_renderer.build_row(case) row[0] = f'[red]- Removed Case[/]\n{row[0]}' table.add_row(*row) if self.include_averages: # pragma: no branch report_average = ReportCaseAggregate.average(report_cases) baseline_average = ReportCaseAggregate.average(baseline_cases) table.add_row(*case_renderer.build_diff_aggregate_row(report_average, baseline_average)) return table