mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 4.8
9c4af4f546
Close the HarnessX self-improving loop without an agent in the loop. Every scan now feeds its verifier outcomes back into the next scan's ranking: - New `calibrate` stage (after rank, before findings) loads a persistent ledger of scoped false-positive learnings and demotes every previously-rejected scope (`calibrate_rankings`), writing `calibration/applied_calibrations.json`. - New `record_calibration` stage (after verify) converts this run's non-accepted verifier outcomes into scoped `FalsePositiveLearning` records (`learnings_from_verifications`) and merges them into `.openultrasast/calibration/false_positive_learnings.json`, de-duplicated by finding ID so a repeat rejection does not compound without bound. - Lower the calibration floor (MIN_CALIBRATED_PRIORITY = 0.1) so accumulated rejections can demote a scope below the natural ranking minimum. Demotion is scoped and reversible: a rejection in one path never touches an accepted finding in another, and priority is demoted rather than the finding deleted so the audit trail explains every downgrade. First run is a no-op (empty ledger), so existing single-run behavior is unchanged. Adds tests/test_pipeline_calibration.py (two-run demotion, order-independent), documents the realized loop in the README, and checks off the standard_security_harness milestone in tasks.md. Co-Authored-By: Claude Opus 4.8 <[email protected]>
66 lines
3.0 KiB
Python
66 lines
3.0 KiB
Python
"""Automatic in-pipeline calibration feedback.
|
|
|
|
A scan that rejects a finding (e.g. an unreachable pattern match) must persist a
|
|
scoped false-positive learning, and the *next* scan on the same target must load
|
|
that learning and demote the rejected scope's ranking without any agent in the
|
|
loop. Accepted (reachable) findings must not be demoted.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openultrasast.cli import main
|
|
|
|
|
|
def _run_dirs(repo: Path) -> set[Path]:
|
|
runs = repo / ".runs"
|
|
return set(runs.iterdir()) if runs.exists() else set()
|
|
|
|
|
|
def _ranking_priority(run_dir: Path, path: str) -> float:
|
|
rankings = json.loads((run_dir / "rank" / "ranking.json").read_text())["rankings"]
|
|
return next(r["priority"] for r in rankings if r["path"] == path)
|
|
|
|
|
|
def _finding_priority(run_dir: Path, path: str) -> float:
|
|
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
|
return next(f["ranking_priority"] for f in findings if f["path"] == path)
|
|
|
|
|
|
def test_rejected_finding_demotes_its_scope_on_the_next_scan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
# Reachable route eval -> accepted (never demoted).
|
|
(repo / "app.py").write_text("@app.route('/x')\ndef view():\n return eval(request.data)\n")
|
|
# Helper eval with no entry point -> unknown reachability -> rejected -> learning.
|
|
(repo / "lib.py").write_text("def helper(x):\n return eval(x)\n")
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
ledger = repo / ".openultrasast" / "calibration" / "false_positive_learnings.json"
|
|
|
|
# --- First scan: produces the learning, applies nothing yet ---------------
|
|
before = _run_dirs(repo)
|
|
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
|
(run1,) = _run_dirs(repo) - before
|
|
learnings = json.loads(ledger.read_text())["learnings"]
|
|
lib_learnings = [item for item in learnings if item["path"] == "lib.py"]
|
|
assert lib_learnings and lib_learnings[0]["reason"] == "unreachable_path"
|
|
assert json.loads((run1 / "calibration" / "applied_calibrations.json").read_text())["calibrations"] == []
|
|
lib_priority_1 = _ranking_priority(run1, "lib.py")
|
|
lib_finding_1 = _finding_priority(run1, "lib.py")
|
|
app_priority_1 = _ranking_priority(run1, "app.py")
|
|
|
|
# --- Second scan: loads the ledger and demotes the rejected scope ---------
|
|
before = _run_dirs(repo)
|
|
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
|
(run2,) = _run_dirs(repo) - before
|
|
applied = json.loads((run2 / "calibration" / "applied_calibrations.json").read_text())["calibrations"]
|
|
applied_paths = {item["path"] for item in applied}
|
|
|
|
assert "lib.py" in applied_paths
|
|
assert "app.py" not in applied_paths # accepted finding is never demoted
|
|
assert _ranking_priority(run2, "lib.py") < lib_priority_1
|
|
assert _finding_priority(run2, "lib.py") < lib_finding_1
|
|
assert _ranking_priority(run2, "app.py") == app_priority_1 # scoped: unrelated path untouched
|