mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(calibration): automatic in-pipeline false-positive feedback
Close the HarnessX self-improving loop without an agent in the loop. Every scan now feeds its verifier outcomes back into the next scan's ranking: - New `calibrate` stage (after rank, before findings) loads a persistent ledger of scoped false-positive learnings and demotes every previously-rejected scope (`calibrate_rankings`), writing `calibration/applied_calibrations.json`. - New `record_calibration` stage (after verify) converts this run's non-accepted verifier outcomes into scoped `FalsePositiveLearning` records (`learnings_from_verifications`) and merges them into `.openultrasast/calibration/false_positive_learnings.json`, de-duplicated by finding ID so a repeat rejection does not compound without bound. - Lower the calibration floor (MIN_CALIBRATED_PRIORITY = 0.1) so accumulated rejections can demote a scope below the natural ranking minimum. Demotion is scoped and reversible: a rejection in one path never touches an accepted finding in another, and priority is demoted rather than the finding deleted so the audit trail explains every downgrade. First run is a no-op (empty ledger), so existing single-run behavior is unchanged. Adds tests/test_pipeline_calibration.py (two-run demotion, order-independent), documents the realized loop in the README, and checks off the standard_security_harness milestone in tasks.md. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
d9eff9aade
commit
9c4af4f546
@@ -7,6 +7,11 @@ from pathlib import Path
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .rank import RankingScore
|
||||
from .verification import VerificationResult, VerificationStatus
|
||||
|
||||
# Calibration may push a repeatedly-rejected scope below the natural ranking
|
||||
# minimum (1.0) so accumulated false positives keep losing priority over runs.
|
||||
MIN_CALIBRATED_PRIORITY = 0.1
|
||||
|
||||
|
||||
class FalsePositiveReason(StrEnum):
|
||||
@@ -139,7 +144,7 @@ def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearni
|
||||
prompt_constraints: list[str] = []
|
||||
retrieval_adjustments: dict[str, str] = {}
|
||||
for learning in applicable:
|
||||
calibrated = max(1.0, calibrated - learning.demotion)
|
||||
calibrated = max(MIN_CALIBRATED_PRIORITY, calibrated - learning.demotion)
|
||||
prompt_constraints.append(
|
||||
f"Do not repeat unsupported {learning.vulnerability_class} claim for scope {learning.scope} without new evidence."
|
||||
)
|
||||
@@ -171,7 +176,7 @@ def apply_ranking_calibration(ranking: RankingScore, calibration: RankingCalibra
|
||||
surface=ranking.surface,
|
||||
influence=ranking.influence,
|
||||
reachability=ranking.reachability,
|
||||
priority=max(1.0, min(ranking.priority, calibration.calibrated_priority)),
|
||||
priority=max(MIN_CALIBRATED_PRIORITY, min(ranking.priority, calibration.calibrated_priority)),
|
||||
rationale=f"{ranking.rationale}; calibrated by false-positive learning" if calibration.applied_learning_ids else ranking.rationale,
|
||||
model_id=ranking.model_id,
|
||||
static_boosts=ranking.static_boosts,
|
||||
@@ -183,6 +188,76 @@ def write_false_positive_learnings(learnings: list[FalsePositiveLearning], path:
|
||||
path.write_text(json.dumps({"learnings": [asdict(learning) for learning in learnings]}, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def write_ranking_calibrations(calibrations: list[RankingCalibration], path: Path) -> None:
|
||||
applied = [calibration for calibration in calibrations if calibration.applied_learning_ids]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"calibrations": [asdict(calibration) for calibration in applied]}, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def load_false_positive_learnings(path: Path) -> list[FalsePositiveLearning]:
|
||||
if not path.exists():
|
||||
return []
|
||||
payload = json.loads(path.read_text())
|
||||
learnings: list[FalsePositiveLearning] = []
|
||||
for item in payload.get("learnings", []):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
learnings.append(
|
||||
FalsePositiveLearning(
|
||||
finding_id=str(item["finding_id"]),
|
||||
path=str(item["path"]),
|
||||
vulnerability_class=str(item["vulnerability_class"]),
|
||||
reason=FalsePositiveReason(item["reason"]),
|
||||
evidence=str(item["evidence"]),
|
||||
scope=str(item["scope"]),
|
||||
demotion=float(item.get("demotion", 0.5)),
|
||||
)
|
||||
)
|
||||
return learnings
|
||||
|
||||
|
||||
def merge_false_positive_learnings(
|
||||
existing: list[FalsePositiveLearning],
|
||||
new: list[FalsePositiveLearning],
|
||||
) -> list[FalsePositiveLearning]:
|
||||
by_id: dict[str, FalsePositiveLearning] = {learning.finding_id: learning for learning in existing}
|
||||
for learning in new:
|
||||
by_id[learning.finding_id] = learning
|
||||
return [by_id[finding_id] for finding_id in sorted(by_id)]
|
||||
|
||||
|
||||
def learnings_from_verifications(
|
||||
findings: list[StaticFinding],
|
||||
verifications: list[VerificationResult],
|
||||
) -> list[FalsePositiveLearning]:
|
||||
finding_by_id = {finding.finding_id: finding for finding in findings}
|
||||
learnings: list[FalsePositiveLearning] = []
|
||||
for result in verifications:
|
||||
if result.status == VerificationStatus.ACCEPTED:
|
||||
continue
|
||||
finding = finding_by_id.get(result.finding_id)
|
||||
if finding is None:
|
||||
continue
|
||||
outcome = FindingOutcome.UNVERIFIED if result.status == VerificationStatus.NEEDS_EVIDENCE else FindingOutcome.REJECTED
|
||||
learning = learning_from_finding_outcome(
|
||||
finding,
|
||||
outcome=outcome,
|
||||
evidence=f"{result.counter_case} {result.required_next_step}".strip(),
|
||||
reason=_reason_from_verification(finding, result.status),
|
||||
)
|
||||
if learning is not None:
|
||||
learnings.append(learning)
|
||||
return learnings
|
||||
|
||||
|
||||
def _reason_from_verification(finding: StaticFinding, status: VerificationStatus) -> FalsePositiveReason:
|
||||
if finding.reachability_status == "unknown":
|
||||
return FalsePositiveReason.UNREACHABLE_PATH
|
||||
if status == VerificationStatus.NEEDS_EVIDENCE:
|
||||
return FalsePositiveReason.UNVERIFIED
|
||||
return FalsePositiveReason.CONTRADICTED
|
||||
|
||||
|
||||
def _vulnerability_class(finding: StaticFinding) -> str:
|
||||
if finding.tags:
|
||||
return finding.tags[0]
|
||||
|
||||
@@ -15,6 +15,15 @@ from .benchmark import (
|
||||
resolve_benchmark_source,
|
||||
write_benchmark_artifacts,
|
||||
)
|
||||
from .calibration import (
|
||||
FalsePositiveLearning,
|
||||
calibrate_rankings,
|
||||
learnings_from_verifications,
|
||||
load_false_positive_learnings,
|
||||
merge_false_positive_learnings,
|
||||
write_false_positive_learnings,
|
||||
write_ranking_calibrations,
|
||||
)
|
||||
from .config import load_config
|
||||
from .findings import StaticFinding, quick_scan_findings, write_findings
|
||||
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||
@@ -24,8 +33,10 @@ from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sar
|
||||
from .preprocess import preprocess_repository, write_preprocess_artifact
|
||||
from .rank import rank_targets, write_rankings
|
||||
from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report
|
||||
from .run import create_scan_run
|
||||
from .verification import verify_findings, write_verification_results
|
||||
from .run import ScanRun, create_scan_run
|
||||
from .verification import VerificationResult, verify_findings, write_verification_results
|
||||
|
||||
CALIBRATION_DIR = ".openultrasast/calibration"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -35,6 +46,7 @@ class ScanOutcome:
|
||||
file_target_count: int
|
||||
ranked_target_count: int
|
||||
finding_count: int
|
||||
calibrations_applied: int
|
||||
exit_code: int
|
||||
|
||||
|
||||
@@ -100,7 +112,12 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
targets = attach_reachability_hints(targets, entry_points)
|
||||
write_preprocess_artifact(snapshot, targets, run.root / "preprocess" / "file_targets.json")
|
||||
rankings = runtime.run_stage("rank", lambda: rank_targets(targets))
|
||||
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
||||
prior_learnings = load_false_positive_learnings(ledger_path)
|
||||
rankings, calibrations = runtime.run_stage("calibrate", lambda: calibrate_rankings(rankings, prior_learnings))
|
||||
applied_calibrations = [calibration for calibration in calibrations if calibration.applied_learning_ids]
|
||||
write_rankings(rankings, run.root / "rank" / "ranking.json")
|
||||
write_ranking_calibrations(calibrations, run.root / "calibration" / "applied_calibrations.json")
|
||||
if mode == "standard":
|
||||
hunter_result = runtime.run_stage("hunter_pool", lambda: run_hunter_pool(run.target, targets, rankings, scan_id=run.scan_id))
|
||||
findings = hunter_result.findings
|
||||
@@ -119,6 +136,10 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
write_hunter_trajectories(trajectories, trajectories_path)
|
||||
verifications = runtime.run_stage("verify", lambda: verify_findings(findings))
|
||||
write_verification_results(verifications, verification_path)
|
||||
runtime.run_stage(
|
||||
"record_calibration",
|
||||
lambda: _persist_calibration_feedback(run, ledger_path, prior_learnings, findings, verifications),
|
||||
)
|
||||
runtime.run_stage("report", lambda: write_markdown_report(findings, markdown_path, verifications))
|
||||
runtime.run_stage("sarif", lambda: write_sarif_report(findings, verifications, sarif_path))
|
||||
runtime.run_stage(
|
||||
@@ -144,16 +165,34 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
file_target_count=len(targets),
|
||||
ranked_target_count=len(rankings),
|
||||
finding_count=len(findings),
|
||||
calibrations_applied=len(applied_calibrations),
|
||||
exit_code=scan_exit_code(findings, verifications, fail_on),
|
||||
)
|
||||
|
||||
|
||||
def _persist_calibration_feedback(
|
||||
run: ScanRun,
|
||||
ledger_path: Path,
|
||||
prior_learnings: list[FalsePositiveLearning],
|
||||
findings: list[StaticFinding],
|
||||
verifications: list[VerificationResult],
|
||||
) -> list[FalsePositiveLearning]:
|
||||
# Turn this run's non-accepted verifier outcomes into scoped learnings and
|
||||
# merge them into the persistent ledger so the next scan demotes those scopes.
|
||||
new_learnings = learnings_from_verifications(findings, verifications)
|
||||
merged = merge_false_positive_learnings(prior_learnings, new_learnings)
|
||||
write_false_positive_learnings(merged, ledger_path)
|
||||
write_false_positive_learnings(new_learnings, run.root / "calibration" / "false_positive_learnings.json")
|
||||
return merged
|
||||
|
||||
|
||||
def _print_scan_outcome(outcome: ScanOutcome) -> None:
|
||||
print(f"scan_id={outcome.scan_id}")
|
||||
print(f"run_dir={outcome.run_dir}")
|
||||
print(f"file_targets={outcome.file_target_count}")
|
||||
print(f"ranked_targets={outcome.ranked_target_count}")
|
||||
print(f"findings={outcome.finding_count}")
|
||||
print(f"calibrations_applied={outcome.calibrations_applied}")
|
||||
|
||||
|
||||
def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]:
|
||||
|
||||
Reference in New Issue
Block a user