fix(learning-harness): the published table lost a whole slice

`publish` keyed baselines by model alone, so the vibe-py run silently
replaced the agent-vfc run and the table showed hand-written teaching code as
if it were the corpus. That is precisely the failure the constrained-detector
requirements exist to prevent, live in a published artifact.

Third time today that a key was fixed in one place and not another --
`compare_baselines` got the model+slice label and `_baselines` did not, the
same way the exception path got checkpointing and the timeout path did not.

The table now carries a slice column and one row per slice, and the artifact
is keyed by what identifies a run. Republished: agent-vfc's six families are
back, including access_control at 0.000 recall on four pairs, which is the
number that was missing.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 20:44:56 +02:00
co-authored by Claude Opus 5
parent 506bbe5f5c
commit 84b1ba7a72
4 changed files with 715 additions and 12 deletions
+38 -1
View File
@@ -153,7 +153,8 @@ def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path:
written = sorted(path.name for path in measurements.iterdir())
assert any(name.endswith("-learning-families.json") for name in written)
payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text())
assert payload["models"]["deepseek-v4-flash"]["injection"]["scorable"] == 10
# keyed by what identifies the run, so two slices of one model cannot collapse into one entry
assert payload["models"]["deepseek-v4-flash (vibe-py)"]["injection"]["scorable"] == 10
assert payload["command"]
@@ -165,3 +166,39 @@ def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_p
report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap)
assert report.families == () and "no baseline" in report.table.lower()
assert "learning-harness:begin" in roadmap.read_text()
def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_path: Path) -> None:
"""Keyed by model alone, the vibe-py run silently replaced the agent-vfc run and the published table showed
hand-written teaching code as if it were the corpus — the exact failure Req 3 exists to prevent."""
from openultrasast.learning.publish import publish
out = tmp_path / "learning"
for key, slice_name, recall in (("deepseek-v4-flash-vibe-py", "vibe-py", 0.833), ("deepseek-v4-flash-agent-vfc", "agent-vfc", 0.444)):
directory = out / "baseline" / key
directory.mkdir(parents=True)
(directory / "report.json").write_text(
json.dumps(
{
"model": "deepseek-v4-flash",
"slices": [slice_name],
"k_runs": 5,
"cost_usd": 1.0,
"taxonomy_version": "1",
"metrics": {
"injection": {
"family": "injection", "scorable": 10, "unscorable": {}, "recall": recall,
"silence": 1.0, "youden": recall, "hierarchical_credit": False,
"outcomes": {"pair_correct": 5},
}
},
"floors": {},
}
)
)
roadmap = tmp_path / "roadmap.md"
roadmap.write_text("# Roadmap\n")
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
assert "vibe-py" in report.table and "agent-vfc" in report.table
assert report.table.count("| injection |") == 2, "one row per slice, not one row that survived"
assert "0.833" in report.table and "0.444" in report.table