fix(learning-harness): a slice is part of what identifies a baseline

Keyed by model alone, the memory-safety run over the vfc slice silently
replaced the run over the web families, and the published table would have
shown one of them as if it were everything. The artifact key and the
comparison label now carry the slice selection when it is not the default.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 14:39:43 +02:00
co-authored by Claude Opus 5
parent c1ef3ca539
commit ce89b0dbe0
4 changed files with 58 additions and 6 deletions
+3 -1
View File
@@ -208,6 +208,8 @@ def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monke
)
== 0
)
report = json.loads((tmp_path / "l" / "baseline" / "m" / "report.json").read_text())
# A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run
# over the web families.
report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text())
assert report["slices"] == ["vfc"]
assert report["metrics"]["injection"]["scorable"] == 1
+37
View File
@@ -258,3 +258,40 @@ def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_r
assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured"
assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1}
assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file()
def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None:
"""Artifacts are keyed by what identifies the run, and the slice selection is part of that.
Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model
alone, the second run silently replaces the first and the published table shows one of them as if it were
everything."""
from openultrasast.learning.rounds import compare_baselines, run_baseline
taxonomy = load_families()
out = tmp_path / "out"
web = run_baseline(
[_case(tmp_path, "w", slice_name="vibe-py")],
taxonomy=taxonomy,
configs_dir=tmp_path / "configs",
scan_factory=_always(_steady),
model="deepseek-v4-flash",
out_dir=out,
k_runs=3,
)
memory = run_baseline(
[_case(tmp_path, "m", family="memory", slice_name="vfc")],
taxonomy=taxonomy,
configs_dir=tmp_path / "configs",
scan_factory=_always(_steady),
model="deepseek-v4-flash",
out_dir=out,
k_runs=3,
slices=("vfc",),
)
assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"}
written = sorted(path.name for path in (out / "baseline").iterdir())
assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written
table = compare_baselines(out)
assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"}
assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"]