mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
fcda1ee46b
Task 3.2. The model-driven hunter is now scored by family and function. A detection is a finding of the labeled family inside the labeled function, a finding of another family or outside it is counted and never called a leak, and the words in a finding are never read. A pair that cannot be scored is reported with its reason and the detector is never run on it. The scoreboard gains a per-family block with its own denominator, taxonomy version and the note that hierarchical partial credit could not apply. The text-token matcher this path used is deleted rather than left dormant. It was the reason the hunter's published number said zero of seventeen while the model was finding the right bugs in the right functions, and leaving it in the file would invite its return. The test that pinned the old rule now pins the new one: the same finding with the same words counts when it carries a family tag and does not when it carries none. The overlay and inventory paths and all three gate outputs are unchanged. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
111 lines
4.8 KiB
Python
111 lines
4.8 KiB
Python
"""learning-harness task 3.2: family scoring on the pairs hunter path (offline)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.findings import StaticFinding
|
|
from openultrasast.pairs import PairCase, evaluate_pair
|
|
|
|
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
|
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', request.args.get('cmd')], check=False)\n"
|
|
|
|
|
|
def _case(tmp_path: Path, *, family: str | None = "injection", unscorable: str | None = None) -> PairCase:
|
|
(tmp_path / "v.py").write_text(VULN)
|
|
(tmp_path / "f.py").write_text(FIXED)
|
|
return PairCase(
|
|
name="p",
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / "v.py",
|
|
fixed_file=tmp_path / "f.py",
|
|
relpath="app.py",
|
|
expected=(
|
|
ExpectedFinding(
|
|
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family
|
|
),
|
|
),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
unscorable=unscorable,
|
|
)
|
|
|
|
|
|
def _finding(family: str, line: int) -> StaticFinding:
|
|
return StaticFinding(
|
|
finding_id=f"detector:app.py:{line}",
|
|
path="app.py",
|
|
title="t",
|
|
severity="high",
|
|
confidence="low",
|
|
evidence_level="suspicion",
|
|
rationale="r",
|
|
line=line,
|
|
function_name="run",
|
|
reachability_status="unknown",
|
|
reachability_evidence=[],
|
|
reachability_conditions=[],
|
|
tags=[f"family:{family}", "detector:injection@1"],
|
|
ranking_priority=0.0,
|
|
)
|
|
|
|
|
|
def _hunter(vuln: list[StaticFinding], fixed: list[StaticFinding]): # type: ignore[no-untyped-def]
|
|
seen: list[Path] = []
|
|
|
|
def scan(root: Path) -> list[StaticFinding]:
|
|
seen.append(root)
|
|
return vuln if root.name == "vuln" else fixed
|
|
|
|
scan.seen = seen # type: ignore[attr-defined]
|
|
return scan
|
|
|
|
|
|
def test_the_hunter_path_scores_by_family_and_function_not_by_text(tmp_path: Path) -> None:
|
|
right = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], []))
|
|
assert right.detected_vuln and right.silent_fix and right.pair_correct
|
|
assert right.detection_kinds == ("hunter",) and right.family == "injection"
|
|
wrong_family = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("access_control", 6)], []))
|
|
assert not wrong_family.detected_vuln # the right place, the wrong family
|
|
loud = StaticFinding(**{**_finding("access_control", 6).__dict__, "title": "CWE-78 command injection in run"})
|
|
assert not evaluate_pair(_case(tmp_path), hunter=_hunter([loud], [])).detected_vuln # the words never count
|
|
|
|
|
|
def test_a_finding_of_another_family_on_the_fixed_side_is_not_a_leak(tmp_path: Path) -> None:
|
|
outcome = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("access_control", 6)]))
|
|
assert outcome.pair_correct and outcome.silent_fix
|
|
assert outcome.fix_findings == 1 and outcome.fix_leaks == 0 # counted, never charged
|
|
leaked = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("injection", 6)]))
|
|
assert leaked.detected_vuln and not leaked.silent_fix and leaked.fix_leaks == 1
|
|
|
|
|
|
def test_an_unscorable_pair_is_reported_as_such_and_never_run(tmp_path: Path) -> None:
|
|
hunter = _hunter([_finding("injection", 6)], [])
|
|
outcome = evaluate_pair(_case(tmp_path, unscorable="identical_twin"), hunter=hunter)
|
|
assert outcome.unscorable == "identical_twin" and not outcome.detected_vuln
|
|
assert hunter.seen == [] # type: ignore[attr-defined]
|
|
|
|
|
|
def test_a_pair_with_no_family_label_falls_back_to_the_classifier(tmp_path: Path) -> None:
|
|
outcome = evaluate_pair(_case(tmp_path, family=None), hunter=_hunter([_finding("injection", 6)], []))
|
|
assert outcome.family == "injection" and outcome.pair_correct # CWE-78 classifies as injection
|
|
|
|
|
|
def test_the_overlay_path_is_untouched_by_the_family_scorer(tmp_path: Path) -> None:
|
|
overlay = evaluate_pair(_case(tmp_path))
|
|
assert overlay.detection_kinds != ("hunter",) and overlay.family == ""
|
|
|
|
|
|
def test_the_scoreboard_carries_the_family_block(tmp_path: Path) -> None:
|
|
from openultrasast.pairs import evaluate_catalog, result_payload
|
|
|
|
result = evaluate_catalog([_case(tmp_path)], hunter=_hunter([_finding("injection", 6)], []), inventory_sidecar=False)
|
|
payload = result_payload(result)
|
|
assert "per_family" in payload and payload["per_family"]["injection"]["scorable"] == 1
|
|
assert payload["per_family"]["injection"]["recall"] == 1.0
|
|
assert payload["per_family"]["injection"]["hierarchical_credit"] is False
|
|
assert payload["per_family"]["injection"]["taxonomy_version"]
|