mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(learning-harness): a slice is part of what identifies a baseline
Keyed by model alone, the memory-safety run over the vfc slice silently replaced the run over the web families, and the published table would have shown one of them as if it were everything. The artifact key and the comparison label now carry the slice selection when it is not the default. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
c1ef3ca539
commit
ce89b0dbe0
@@ -208,6 +208,8 @@ def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monke
|
||||
)
|
||||
== 0
|
||||
)
|
||||
report = json.loads((tmp_path / "l" / "baseline" / "m" / "report.json").read_text())
|
||||
# A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run
|
||||
# over the web families.
|
||||
report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text())
|
||||
assert report["slices"] == ["vfc"]
|
||||
assert report["metrics"]["injection"]["scorable"] == 1
|
||||
|
||||
@@ -258,3 +258,40 @@ def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_r
|
||||
assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured"
|
||||
assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1}
|
||||
assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file()
|
||||
|
||||
|
||||
def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None:
|
||||
"""Artifacts are keyed by what identifies the run, and the slice selection is part of that.
|
||||
|
||||
Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model
|
||||
alone, the second run silently replaces the first and the published table shows one of them as if it were
|
||||
everything."""
|
||||
from openultrasast.learning.rounds import compare_baselines, run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
out = tmp_path / "out"
|
||||
web = run_baseline(
|
||||
[_case(tmp_path, "w", slice_name="vibe-py")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="deepseek-v4-flash",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
)
|
||||
memory = run_baseline(
|
||||
[_case(tmp_path, "m", family="memory", slice_name="vfc")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="deepseek-v4-flash",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
slices=("vfc",),
|
||||
)
|
||||
assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"}
|
||||
written = sorted(path.name for path in (out / "baseline").iterdir())
|
||||
assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written
|
||||
table = compare_baselines(out)
|
||||
assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"}
|
||||
assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"]
|
||||
|
||||
Reference in New Issue
Block a user