From ce89b0dbe059684ab631192bc4dc6901049ba969 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:39:43 +0200 Subject: [PATCH] fix(learning-harness): a slice is part of what identifies a baseline Keyed by model alone, the memory-safety run over the vfc slice silently replaced the run over the web families, and the published table would have shown one of them as if it were everything. The artifact key and the comparison label now carry the slice selection when it is not the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- src/openultrasast/cli.py | 2 +- src/openultrasast/learning/rounds.py | 21 +++++++++++++--- tests/test_learning_cutoff.py | 4 ++- tests/test_learning_rounds.py | 37 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 58d77cf..77f1f59 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -1343,7 +1343,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: model=model, journal=journal, archive=Archive(out / "archive.jsonl"), - floors=load_noise_floors(out, model), + floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES), out_dir=out, cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd, minibatch=config.learning.minibatch, diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py index 6f6e482..0815690 100644 --- a/src/openultrasast/learning/rounds.py +++ b/src/openultrasast/learning/rounds.py @@ -196,7 +196,10 @@ def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]: if not report.is_file(): continue payload = json.loads(report.read_text(encoding="utf-8")) - table[str(payload.get("model", directory.name))] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} + model = str(payload.get("model", directory.name)) + slices = tuple(str(item) for item in (payload.get("slices") or ())) + label = model if not slices or slices == DEFAULT_SLICES else f"{model} ({', '.join(slices)})" + table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} return table @@ -224,8 +227,18 @@ def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str: return answer.families[0] if answer.families else "unknown" +def _artifact_key(model: str, slices: Sequence[str]) -> str: + """What identifies a baseline run: the detector model, and the slice selection when it is not the default. + + Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and + the published table shows one of them as if it were everything (Req 8.4).""" + if tuple(slices) == DEFAULT_SLICES: + return _slug(model) + return f"{_slug(model)}-{_slug('-'.join(sorted(slices)))}" + + def _write(out_dir: Path, report: BaselineReport) -> None: - directory = out_dir / "baseline" / _slug(report.model) + directory = out_dir / "baseline" / _artifact_key(report.model, report.slices) directory.mkdir(parents=True, exist_ok=True) payload = report.to_dict() (directory / "noise-floors.json").write_text( @@ -239,9 +252,9 @@ def _slug(model: str) -> str: return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed" -def load_noise_floors(out_dir: Path, model: str) -> dict[str, NoiseFloor]: +def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = ()) -> dict[str, NoiseFloor]: """The budgets a later round must stay inside, as round zero measured them for this model.""" - path = out_dir / "baseline" / _slug(model) / "noise-floors.json" + path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES) / "noise-floors.json" if not path.is_file(): return {} payload = json.loads(path.read_text(encoding="utf-8")) diff --git a/tests/test_learning_cutoff.py b/tests/test_learning_cutoff.py index b4c7b91..f22fd3d 100644 --- a/tests/test_learning_cutoff.py +++ b/tests/test_learning_cutoff.py @@ -208,6 +208,8 @@ def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monke ) == 0 ) - report = json.loads((tmp_path / "l" / "baseline" / "m" / "report.json").read_text()) + # A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run + # over the web families. + report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text()) assert report["slices"] == ["vfc"] assert report["metrics"]["injection"]["scorable"] == 1 diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py index be0a148..0b4d827 100644 --- a/tests/test_learning_rounds.py +++ b/tests/test_learning_rounds.py @@ -258,3 +258,40 @@ def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_r assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured" assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1} assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file() + + +def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: + """Artifacts are keyed by what identifies the run, and the slice selection is part of that. + + Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model + alone, the second run silently replaces the first and the published table shows one of them as if it were + everything.""" + from openultrasast.learning.rounds import compare_baselines, run_baseline + + taxonomy = load_families() + out = tmp_path / "out" + web = run_baseline( + [_case(tmp_path, "w", slice_name="vibe-py")], + taxonomy=taxonomy, + configs_dir=tmp_path / "configs", + scan_factory=_always(_steady), + model="deepseek-v4-flash", + out_dir=out, + k_runs=3, + ) + memory = run_baseline( + [_case(tmp_path, "m", family="memory", slice_name="vfc")], + taxonomy=taxonomy, + configs_dir=tmp_path / "configs", + scan_factory=_always(_steady), + model="deepseek-v4-flash", + out_dir=out, + k_runs=3, + slices=("vfc",), + ) + assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"} + written = sorted(path.name for path in (out / "baseline").iterdir()) + assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written + table = compare_baselines(out) + assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"} + assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"]