mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
b0dba26468
Task 2.3: two of the five unscorable reasons had no code path, so an unparsable row was reported as a corpus labelling defect when the true cause was a tooling gap. They have different owners and different fixes, and conflating them is the dishonesty this spec exists to remove. The design's unscorable_reason is reinstated with the parse result as an input, and a handler that arrived without its registration is now named as missing context for the families whose bug shape is a missing check. Task 2.6: a verification stamped its own word for proof onto the shared evidence level, which is not a value the project's ladder defines, and three existing helpers raised on it. Nothing in production called it yet, so the suite stayed green while the wiring in the next group would have broken a scan. A verification now writes only its tag, and a caller that wants to move a claim up the ladder goes through an explicit mapping that can only produce a level the ladder has. Task 2.7: the injection canary planted its row in a private in-memory database the target could never reach, so it could not fire; the fake sandbox handed the token back regardless of the snippet, which is why nothing noticed. It is now an execution canary whose marker only an interpreter could write, and each template is tested by running it for real against a vulnerable fixture and its fixed twin. That test immediately caught a second flaw: a handler that merely echoes its input printed the oracle token and satisfied the oracle without any vulnerability, so an injected payload now carries a value unrelated to the token the oracle looks for. Also from the review: a configuration with no tools is refused instead of producing a detector that silently reports nothing, the classifier abstains when its fallback call fails too, and the archive docstring says which version wins. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
186 lines
9.0 KiB
Python
186 lines
9.0 KiB
Python
"""learning-harness task 2.3: per-pair family scoring (offline)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.findings import StaticFinding
|
|
from openultrasast.learning.families import load_families
|
|
from openultrasast.pairs import PairCase
|
|
|
|
RANGES = {"app.py": (("leaky", 10, 13), ("guarded", 4, 7))}
|
|
|
|
|
|
def _finding(family: str, line: int, path: str = "app.py") -> StaticFinding:
|
|
return StaticFinding(
|
|
finding_id=f"detector:{path}:{line}",
|
|
path=path,
|
|
title="t",
|
|
severity="medium",
|
|
confidence="low",
|
|
evidence_level="suspicion",
|
|
rationale="r",
|
|
line=line,
|
|
function_name=None,
|
|
reachability_status="unknown",
|
|
reachability_evidence=[],
|
|
reachability_conditions=[],
|
|
tags=[f"family:{family}", "detector:x@1"],
|
|
ranking_priority=0.0,
|
|
)
|
|
|
|
|
|
def _case(tmp_path: Path, *, family: str = "access_control", unscorable: str | None = None) -> PairCase:
|
|
(tmp_path / "v.py").write_text("x = 1\n")
|
|
(tmp_path / "f.py").write_text("x = 2\n")
|
|
return PairCase(
|
|
name="p",
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / "v.py",
|
|
fixed_file=tmp_path / "f.py",
|
|
relpath="app.py",
|
|
expected=(
|
|
ExpectedFinding(
|
|
cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function="leaky", mechanism="other", family=family
|
|
),
|
|
),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
unscorable=unscorable,
|
|
)
|
|
|
|
|
|
def test_detection_needs_the_labeled_family_inside_the_labeled_function(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
case = _case(tmp_path)
|
|
hit = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert hit.outcome == "pair_correct" and hit.runs == ("pair_correct",)
|
|
outside = score_pair_family(case, "access_control", [[_finding("access_control", 5)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert outside.outcome == "both_silent" # the right family in the wrong function is not a detection
|
|
other_file = score_pair_family(
|
|
case, "access_control", [[_finding("access_control", 12, "other.py")]], [[]], ranges=RANGES, taxonomy=taxonomy
|
|
)
|
|
assert other_file.outcome == "both_silent"
|
|
wrong_family = score_pair_family(case, "access_control", [[_finding("injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert wrong_family.outcome == "both_silent" and wrong_family.other_findings_vuln == 1
|
|
|
|
|
|
def test_a_fabricated_family_never_counts_and_is_recorded(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
score = score_pair_family(_case(tmp_path), "access_control", [[_finding("sql_injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert score.outcome == "both_silent" and score.fabricated_vuln == 1
|
|
assert score.other_findings_vuln == 0 # a fabricated id is not another family, it is noise
|
|
|
|
|
|
def test_silence_ignores_findings_of_other_families_and_never_calls_them_leaks(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
case = _case(tmp_path)
|
|
quiet = score_pair_family(
|
|
case,
|
|
"access_control",
|
|
[[_finding("access_control", 12)]],
|
|
[[_finding("injection", 12), _finding("access_control", 5)]],
|
|
ranges=RANGES,
|
|
taxonomy=taxonomy,
|
|
)
|
|
assert quiet.outcome == "pair_correct" and quiet.other_findings_fixed == 2
|
|
leaked = score_pair_family(
|
|
case, "access_control", [[_finding("access_control", 12)]], [[_finding("access_control", 13)]], ranges=RANGES, taxonomy=taxonomy
|
|
)
|
|
assert leaked.outcome == "both_flagged"
|
|
reversed_pair = score_pair_family(case, "access_control", [[]], [[_finding("access_control", 12)]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert reversed_pair.outcome == "reversed"
|
|
|
|
|
|
def test_the_outcome_of_several_runs_is_their_majority(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
case = _case(tmp_path)
|
|
hit = [_finding("access_control", 12)]
|
|
score = score_pair_family(case, "access_control", [hit, [], hit], [[], [], []], ranges=RANGES, taxonomy=taxonomy)
|
|
assert score.runs == ("pair_correct", "both_silent", "pair_correct") and score.outcome == "pair_correct"
|
|
flaky = score_pair_family(case, "access_control", [hit, [], []], [[], [], []], ranges=RANGES, taxonomy=taxonomy)
|
|
assert flaky.outcome == "both_silent" and flaky.flips == 1 # a run that disagrees with the majority is a flip
|
|
|
|
|
|
def test_an_unscorable_pair_is_never_run_and_keeps_its_reason(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
case = _case(tmp_path, unscorable="identical_twin")
|
|
score = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert score.outcome == "unscorable" and score.unscorable_reason == "identical_twin" and score.runs == ()
|
|
|
|
|
|
def test_a_pair_with_no_parsed_function_range_is_unscorable_not_a_miss(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy)
|
|
assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label"
|
|
|
|
|
|
def test_scoring_never_reads_the_text_of_a_finding(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
shouting = _finding("injection", 12)
|
|
loud = StaticFinding(**{**shouting.__dict__, "title": "CWE-639 broken object level authorization", "rationale": "IDOR in leaky"})
|
|
score = score_pair_family(_case(tmp_path), "access_control", [[loud]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert score.outcome == "both_silent" # the words say access control; the family tag says injection, and only that counts
|
|
|
|
|
|
def test_the_rung_is_the_best_verifier_outcome_among_the_detections(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
corroborated = _finding("access_control", 12)
|
|
corroborated.tags.append("verifier:static_corroboration")
|
|
score = score_pair_family(_case(tmp_path), "access_control", [[corroborated]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert score.outcome == "pair_correct" and score.rung == "static_corroboration"
|
|
plain = score_pair_family(_case(tmp_path), "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
|
assert plain.rung == "suspicion"
|
|
|
|
|
|
def test_the_unscorable_reason_tells_a_tooling_gap_from_a_corpus_defect(tmp_path: Path) -> None:
|
|
"""Req 4.1: an unparsable row is a tooling gap; a row whose labeled function no range names is a corpus defect."""
|
|
from openultrasast.learning.scoring import unscorable_reason
|
|
|
|
case = _case(tmp_path)
|
|
assert unscorable_reason(case, parse_ok=True, ranges=RANGES) is None
|
|
assert unscorable_reason(case, parse_ok=False, ranges={}) == "unsupported_language"
|
|
assert unscorable_reason(case, parse_ok=False, ranges=RANGES) == "unsupported_language" # the parse decides, not the ranges
|
|
assert unscorable_reason(case, parse_ok=True, ranges={}) == "unresolved_label"
|
|
assert unscorable_reason(_case(tmp_path, unscorable="identical_twin"), parse_ok=True, ranges=RANGES) == "identical_twin"
|
|
twin = _case(tmp_path, unscorable="known_limit:java hash")
|
|
assert unscorable_reason(twin, parse_ok=False, ranges={}) == "known_limit:java hash" # a declared limit wins
|
|
|
|
|
|
def test_a_row_whose_registration_context_is_missing_is_named_as_such(tmp_path: Path) -> None:
|
|
"""Req 4.1: an access-control row whose handler arrived without its registration cannot be scored for that reason."""
|
|
from openultrasast.learning.scoring import unscorable_reason
|
|
|
|
case = _case(tmp_path, family="access_control")
|
|
assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=("leaky",)) is None
|
|
assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=()) == "missing_context"
|
|
# only families whose bug shape needs the registration are judged on it
|
|
assert unscorable_reason(_case(tmp_path, family="injection"), parse_ok=True, ranges=RANGES, entry_points=()) is None
|
|
|
|
|
|
def test_an_unparsable_side_is_scored_unscorable_rather_than_missed(tmp_path: Path) -> None:
|
|
from openultrasast.learning.scoring import score_pair_family
|
|
|
|
taxonomy = load_families()
|
|
score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy, parse_ok=False)
|
|
assert score.outcome == "unscorable" and score.unscorable_reason == "unsupported_language"
|