class ErrorReport:
"""Create box plots of estimate/exact ratios across prediction requests."""
def __init__(self, log: ErrorLog):
self.log = log
def plot(self, filename: str, *, max_ratio: float | None = 10.0) -> None:
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter, LogLocator, NullFormatter
if max_ratio is not None and (not np.isfinite(max_ratio) or max_ratio <= 1):
raise ValueError("max_ratio must be finite and greater than 1, or None")
estimate_ratios = []
labels = []
for factory, samples in self.log.samples.items():
ratios = []
for exact, estimate in samples:
if np.isnan(exact) or np.isnan(estimate) or exact < 0 or estimate < 0:
ratio = float("nan")
elif exact == 0:
ratio = 1.0 if estimate == 0 else float("inf")
else:
ratio = estimate / exact
ratios.append(ratio)
ratios = np.array(ratios)
estimate_ratios.append(ratios[np.isfinite(ratios) & (ratios > 0)])
names = []
factories = [factory]
for inner_factory in factories:
name = (
type(inner_factory)
.__name__.removesuffix("Factory")
.removesuffix("Stats")
)
match inner_factory:
case BlockedStatsFactory(inner_factory=inner):
factories.append(inner)
case SamplingStatsFactory(estimator=estimator):
name = f"{name} ({estimator})"
names.append(name)
name = " ".join(names)
counts = [f"n={len(samples)}"]
for mask, label in (
(ratios == 0, "zero"),
(np.isposinf(ratios), "∞"),
(np.isnan(ratios), "invalid"),
):
count = np.count_nonzero(mask)
if count:
counts.append(f"{label}={count}")
labels.append(f"{name} ({', '.join(counts)})")
minimum = min(
(ratio for ratios in estimate_ratios for ratio in ratios),
default=1.0,
)
maximum = max(
(ratio for ratios in estimate_ratios for ratio in ratios),
default=1.0,
)
right_limit = max(2.0, maximum * 1.2)
if max_ratio is not None:
right_limit = min(right_limit, max_ratio)
figure, axis = plt.subplots(figsize=(10, max(3, 0.4 * len(labels) + 2.5)))
if estimate_ratios:
artists = axis.boxplot(
estimate_ratios,
orientation="horizontal",
tick_labels=labels,
patch_artist=True,
whis=(0, 100),
showfliers=False,
boxprops={"facecolor": "none"},
medianprops={"color": "black"},
)
for box in artists["boxes"]:
bounds = box.get_path().vertices
if not np.isfinite(bounds).all():
continue
left, bottom = bounds.min(axis=0)
right, top = bounds.max(axis=0)
if left < 1:
axis.fill_betweenx(
[bottom, top],
left,
min(right, 1),
color="tab:blue",
alpha=0.35,
zorder=1,
)
if right > 1:
axis.fill_betweenx(
[bottom, top],
max(left, 1),
right,
color="tab:orange",
alpha=0.35,
zorder=1,
)
box.set_edgecolor(
"tab:blue"
if right < 1
else "tab:orange"
if left > 1
else "tab:gray"
)
for line in artists["whiskers"] + artists["caps"] + artists["medians"]:
values = line.get_xdata()
line.set_color(
"tab:blue"
if max(values) < 1
else "tab:orange"
if min(values) > 1
else "tab:gray"
)
for position, ratios in enumerate(estimate_ratios, start=1):
if not len(ratios):
axis.text(
1,
position,
"No positive finite ratios",
ha="center",
va="center",
)
else:
axis.set_yticks([])
axis.text(
0.5,
0.5,
"No predictions recorded",
transform=axis.transAxes,
ha="center",
va="center",
)
axis.set_xscale("log")
axis.set_xlim(min(0.5, minimum / 1.2), right_limit)
for position, ratios in enumerate(estimate_ratios, start=1):
if not len(ratios) or ratios.max() <= right_limit:
continue
axis.plot(
right_limit,
position,
marker=">",
color="tab:orange",
markersize=6,
clip_on=False,
)
axis.annotate(
f"{ratios.max():,.6g}×",
(right_limit, position),
xytext=(8, 0),
textcoords="offset points",
ha="left",
va="center",
color="tab:orange",
annotation_clip=False,
)
axis.xaxis.set_major_locator(LogLocator(base=10, subs=(1, 2, 5)))
axis.xaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{value:g}"))
axis.xaxis.set_minor_formatter(NullFormatter())
axis.axvline(1, color="black", linewidth=0.8)
axis.set_axisbelow(True)
axis.grid(axis="x", which="both", alpha=0.25)
axis.set_xlabel(
"Estimate / exact (log scale; 1 = exact)\n"
"Blue < 1: underestimation; orange > 1: overestimation"
)
axis.set_ylabel("Estimator\n(n = number of prediction requests)")
axis.set_title("Estimate / Exact Distributions")
figure.savefig(filename, bbox_inches="tight")
plt.close(figure)