fix(learning-harness): the published table lost a whole slice

`publish` keyed baselines by model alone, so the vibe-py run silently
replaced the agent-vfc run and the table showed hand-written teaching code as
if it were the corpus. That is precisely the failure the constrained-detector
requirements exist to prevent, live in a published artifact.

Third time today that a key was fixed in one place and not another --
`compare_baselines` got the model+slice label and `_baselines` did not, the
same way the exception path got checkpointing and the timeout path did not.

The table now carries a slice column and one row per slice, and the artifact
is keyed by what identifies a run. Republished: agent-vfc's six families are
back, including access_control at 0.000 recall on four pairs, which is the
number that was missing.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 20:44:56 +02:00
co-authored by Claude Opus 5
parent 506bbe5f5c
commit 84b1ba7a72
4 changed files with 715 additions and 12 deletions
+24 -9
View File
@@ -28,10 +28,11 @@ class Baseline:
"""One model's round-zero artifact, read back."""
model: str
metrics: dict[str, Mapping[str, object]]
floors: dict[str, Mapping[str, object]]
cost_usd: float
k_runs: int
slices: tuple[str, ...] = ()
metrics: dict[str, Mapping[str, object]] = field(default_factory=dict)
floors: dict[str, Mapping[str, object]] = field(default_factory=dict)
cost_usd: float = 0.0
k_runs: int = 0
@dataclass(frozen=True)
@@ -74,6 +75,11 @@ def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date:
return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts))
def _label(model: str, slices: tuple[str, ...]) -> str:
"""What identifies a run in the published table: the model, and the slice when it is not the default pair."""
return f"{model} ({', '.join(slices)})" if slices else model
def _baselines(learning_dir: Path) -> dict[str, Baseline]:
root = learning_dir / "baseline"
found: dict[str, Baseline] = {}
@@ -82,9 +88,13 @@ def _baselines(learning_dir: Path) -> dict[str, Baseline]:
if not report.is_file():
continue
payload = json.loads(report.read_text(encoding="utf-8"))
# Keyed by model *and* slice. Keyed by model alone, a second slice silently replaced the first and the
# published table showed one of them as if it were the corpus (Req 3.1, 3.2).
model = str(payload.get("model", directory.name))
found[model] = Baseline(
slices = tuple(str(item) for item in (payload.get("slices") or ()))
found[_label(model, slices)] = Baseline(
model=model,
slices=slices,
metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)},
floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)},
cost_usd=float(payload.get("cost_usd", 0.0) or 0.0),
@@ -110,10 +120,15 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map
lines = [
f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.",
"",
"| model | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair |",
"|---|---|---|---|---|---|---|---|---|---|",
(
"| model | slice | family | scorable | unscorable | recall | silence | Youden "
"| at fixed FPR | noise floor | cost per correct pair |"
),
"|---|---|---|---|---|---|---|---|---|---|---|",
]
for model, baseline in sorted(models.items()):
for _label_key, baseline in sorted(models.items()):
model = baseline.model
slice_name = ", ".join(baseline.slices) or "all"
for family in families:
row = baseline.metrics.get(family)
if row is None:
@@ -126,7 +141,7 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map
budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0
cost = baseline.cost_usd
lines.append(
f"| {model} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) "
f"| {model} | {slice_name} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) "
f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} "
f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |"
)