mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
47 lines
2.3 KiB
Python
47 lines
2.3 KiB
Python
|
"""Checks that search scoring follows the benchmark's location rule."""
|
||
|
|
|
||
|
|
from types import SimpleNamespace
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from benchmarks.unseen.score import catches
|
||
|
|
from openultrasast.hunt.metric import search_metric, search_outcome
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("change", [
|
||
|
|
{"family": "injection", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
|
||
|
|
{"family": "path", "path": "other.py", "function": "read", "function_lines": [100, 140]},
|
||
|
|
{"family": "config_secrets", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
|
||
|
|
])
|
||
|
|
def test_matches_score_catches(change):
|
||
|
|
change = {**change, "kind": "vulnerability", "files": [{"path": "app.py", "head_lines": [[20, 22]]}]}
|
||
|
|
for path in ("app.py", "other.py"):
|
||
|
|
for line in (9, 10, 22, 32, 33, 110, 500):
|
||
|
|
for function in ("handler", "other"):
|
||
|
|
for family in (change["family"], "unknown"):
|
||
|
|
finding = {"path": path, "line": line, "function": function, "family": family}
|
||
|
|
record = {**change, "push": {"admission": {"defects": [
|
||
|
|
{**finding, "locations": [[path, line]]}
|
||
|
|
]}}}
|
||
|
|
assert search_outcome(change, [finding]).hit == bool(catches(record))
|
||
|
|
|
||
|
|
|
||
|
|
def test_counts_scope_feedback_and_tolerance():
|
||
|
|
change = {"kind": "vulnerability", "path": "app.py", "function": "handle", "family": "path",
|
||
|
|
"function_lines": [20, 22], "files": [{"path": "app.py", "head_lines": [[20, 22]]}]}
|
||
|
|
findings = [{"path": "app.py", "line": 10, "family": "path"},
|
||
|
|
{"path": "other.py", "line": 30, "family": "path"}]
|
||
|
|
outcome = search_outcome(change, findings)
|
||
|
|
assert (outcome.hit, outcome.suspicions, outcome.on_diff) == (True, 2, 1)
|
||
|
|
assert "other.py:30" in outcome.feedback
|
||
|
|
missed = search_outcome(change, findings, tolerance=0)
|
||
|
|
assert not missed.hit
|
||
|
|
assert "missed app.py:20 handle family path" in missed.feedback
|
||
|
|
assert search_metric({"change": change}, {"findings": findings}) == 1.0
|
||
|
|
assert search_metric(SimpleNamespace(change=change), SimpleNamespace(findings=[])) == 0.0
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("count,expected", [(0, 1.0), (1, 0.8), (5, 0.0), (7, 0.0)])
|
||
|
|
def test_ordinary_metric(count, expected):
|
||
|
|
assert search_metric({"kind": "ordinary", "files": []}, {"findings": [{}] * count}) == expected
|