mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(learning-harness): a baseline over a named slice measures that slice
`--slice vfc` selected the vfc rows and `run_baseline` filtered them all out again against its own default of the two web slices, so the memory-safety baseline Req 8.4 asks for produced an empty report and said nothing about it. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
3acad97297
commit
32f16f78ff
@@ -1257,7 +1257,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
|
||||
from .learning.detectors import load_family_configs, write_default_configs
|
||||
from .learning.endpoint import resolve_chat_endpoint, resolve_models
|
||||
from .learning.journal import Archive, LearningJournal
|
||||
from .learning.rounds import MIN_K_RUNS, load_noise_floors, run_baseline, run_learning_round
|
||||
from .learning.rounds import DEFAULT_SLICES, MIN_K_RUNS, load_noise_floors, run_baseline, run_learning_round
|
||||
from .learning.scoring import aggregate
|
||||
from .tool_hunter import _SYSTEM_PROMPT
|
||||
|
||||
@@ -1320,6 +1320,9 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
|
||||
out_dir=out,
|
||||
k_runs=k_runs,
|
||||
cutoff=config.learning.cutoff_date or "",
|
||||
# `--slice` already chose the rows; without this the baseline's own default filters them out again and
|
||||
# a run over the memory-safety slice reports nothing at all (Req 8.4).
|
||||
slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES,
|
||||
)
|
||||
print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}")
|
||||
return 0
|
||||
|
||||
@@ -171,3 +171,43 @@ def test_the_command_line_reports_the_slice_the_cutoff_and_the_undated_count(tmp
|
||||
)
|
||||
line = next(item for item in captured if item.startswith("learning score injection"))
|
||||
assert "post-cutoff(2025-07-01)=2" in line and "undated=0" in line
|
||||
|
||||
|
||||
def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`--slice vfc` selected the vfc rows and then `run_baseline` filtered them all out again.
|
||||
|
||||
Its `slices` default is the two web slices, so a baseline over the memory-safety slice — the run Req 8.4 asks
|
||||
for as a measured, non-gating number — produced an empty report and said nothing about it."""
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
(tmp_path / "a-v.py").write_text(VULN)
|
||||
(tmp_path / "a-f.py").write_text(FIXED)
|
||||
(tmp_path / "pairs.toml").write_text(CATALOG.format(name="a", split="train").replace('slice = "vibe-py"', 'slice = "vfc"'))
|
||||
_detector(monkeypatch)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(tmp_path / "pairs.toml"),
|
||||
"--slice",
|
||||
"vfc",
|
||||
"--out",
|
||||
str(tmp_path / "l"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"m",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
report = json.loads((tmp_path / "l" / "baseline" / "m" / "report.json").read_text())
|
||||
assert report["slices"] == ["vfc"]
|
||||
assert report["metrics"]["injection"]["scorable"] == 1
|
||||
|
||||
Reference in New Issue
Block a user