mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Fable 5.1
0d7d3f0dc9
CI runs ruff format --check and ruff check over the whole repository; 29 files had drifted from the formatter and
four measurement scripts carried lint findings, so every push since 25ae4b4 failed before pytest ran.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
53 lines
2.3 KiB
Python
53 lines
2.3 KiB
Python
"""Checks that search scoring follows the benchmark's location rule."""
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from benchmarks.unseen.score import catches
|
|
from openultrasast.hunt.metric import search_metric, search_outcome
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
[
|
|
{"family": "injection", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
|
|
{"family": "path", "path": "other.py", "function": "read", "function_lines": [100, 140]},
|
|
{"family": "config_secrets", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
|
|
],
|
|
)
|
|
def test_matches_score_catches(change):
|
|
change = {**change, "kind": "vulnerability", "files": [{"path": "app.py", "head_lines": [[20, 22]]}]}
|
|
for path in ("app.py", "other.py"):
|
|
for line in (9, 10, 22, 32, 33, 110, 500):
|
|
for function in ("handler", "other"):
|
|
for family in (change["family"], "unknown"):
|
|
finding = {"path": path, "line": line, "function": function, "family": family}
|
|
record = {**change, "push": {"admission": {"defects": [{**finding, "locations": [[path, line]]}]}}}
|
|
assert search_outcome(change, [finding]).hit == bool(catches(record))
|
|
|
|
|
|
def test_counts_scope_feedback_and_tolerance():
|
|
change = {
|
|
"kind": "vulnerability",
|
|
"path": "app.py",
|
|
"function": "handle",
|
|
"family": "path",
|
|
"function_lines": [20, 22],
|
|
"files": [{"path": "app.py", "head_lines": [[20, 22]]}],
|
|
}
|
|
findings = [{"path": "app.py", "line": 10, "family": "path"}, {"path": "other.py", "line": 30, "family": "path"}]
|
|
outcome = search_outcome(change, findings)
|
|
assert (outcome.hit, outcome.suspicions, outcome.on_diff) == (True, 2, 1)
|
|
assert "other.py:30" in outcome.feedback
|
|
missed = search_outcome(change, findings, tolerance=0)
|
|
assert not missed.hit
|
|
assert "missed app.py:20 handle family path" in missed.feedback
|
|
assert search_metric({"change": change}, {"findings": findings}) == 1.0
|
|
assert search_metric(SimpleNamespace(change=change), SimpleNamespace(findings=[])) == 0.0
|
|
|
|
|
|
@pytest.mark.parametrize("count,expected", [(0, 1.0), (1, 0.8), (5, 0.0), (7, 0.0)])
|
|
def test_ordinary_metric(count, expected):
|
|
assert search_metric({"kind": "ordinary", "files": []}, {"findings": [{}] * count}) == expected
|