diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 58d77cf..77f1f59 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -1343,7 +1343,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: model=model, journal=journal, archive=Archive(out / "archive.jsonl"), - floors=load_noise_floors(out, model), + floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES), out_dir=out, cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd, minibatch=config.learning.minibatch, diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py index 6f6e482..0815690 100644 --- a/src/openultrasast/learning/rounds.py +++ b/src/openultrasast/learning/rounds.py @@ -196,7 +196,10 @@ def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]: if not report.is_file(): continue payload = json.loads(report.read_text(encoding="utf-8")) - table[str(payload.get("model", directory.name))] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} + model = str(payload.get("model", directory.name)) + slices = tuple(str(item) for item in (payload.get("slices") or ())) + label = model if not slices or slices == DEFAULT_SLICES else f"{model} ({', '.join(slices)})" + table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} return table @@ -224,8 +227,18 @@ def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str: return answer.families[0] if answer.families else "unknown" +def _artifact_key(model: str, slices: Sequence[str]) -> str: + """What identifies a baseline run: the detector model, and the slice selection when it is not the default. + + Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and + the published table shows one of them as if it were everything (Req 8.4).""" + if tuple(slices) == DEFAULT_SLICES: + return _slug(model) + return f"{_slug(model)}-{_slug('-'.join(sorted(slices)))}" + + def _write(out_dir: Path, report: BaselineReport) -> None: - directory = out_dir / "baseline" / _slug(report.model) + directory = out_dir / "baseline" / _artifact_key(report.model, report.slices) directory.mkdir(parents=True, exist_ok=True) payload = report.to_dict() (directory / "noise-floors.json").write_text( @@ -239,9 +252,9 @@ def _slug(model: str) -> str: return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed" -def load_noise_floors(out_dir: Path, model: str) -> dict[str, NoiseFloor]: +def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = ()) -> dict[str, NoiseFloor]: """The budgets a later round must stay inside, as round zero measured them for this model.""" - path = out_dir / "baseline" / _slug(model) / "noise-floors.json" + path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES) / "noise-floors.json" if not path.is_file(): return {} payload = json.loads(path.read_text(encoding="utf-8")) diff --git a/tests/test_learning_cutoff.py b/tests/test_learning_cutoff.py index b4c7b91..f22fd3d 100644 --- a/tests/test_learning_cutoff.py +++ b/tests/test_learning_cutoff.py @@ -208,6 +208,8 @@ def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monke ) == 0 ) - report = json.loads((tmp_path / "l" / "baseline" / "m" / "report.json").read_text()) + # A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run + # over the web families. + report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text()) assert report["slices"] == ["vfc"] assert report["metrics"]["injection"]["scorable"] == 1 diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py index be0a148..0b4d827 100644 --- a/tests/test_learning_rounds.py +++ b/tests/test_learning_rounds.py @@ -258,3 +258,40 @@ def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_r assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured" assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1} assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file() + + +def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: + """Artifacts are keyed by what identifies the run, and the slice selection is part of that. + + Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model + alone, the second run silently replaces the first and the published table shows one of them as if it were + everything.""" + from openultrasast.learning.rounds import compare_baselines, run_baseline + + taxonomy = load_families() + out = tmp_path / "out" + web = run_baseline( + [_case(tmp_path, "w", slice_name="vibe-py")], + taxonomy=taxonomy, + configs_dir=tmp_path / "configs", + scan_factory=_always(_steady), + model="deepseek-v4-flash", + out_dir=out, + k_runs=3, + ) + memory = run_baseline( + [_case(tmp_path, "m", family="memory", slice_name="vfc")], + taxonomy=taxonomy, + configs_dir=tmp_path / "configs", + scan_factory=_always(_steady), + model="deepseek-v4-flash", + out_dir=out, + k_runs=3, + slices=("vfc",), + ) + assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"} + written = sorted(path.name for path in (out / "baseline").iterdir()) + assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written + table = compare_baselines(out) + assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"} + assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"]