Open_Ultra_SAST/tests/test_hunt_metric.py

"""Checks that search scoring follows the benchmark's location rule."""

from types import SimpleNamespace

import pytest

from benchmarks.unseen.score import catches
from openultrasast.hunt.metric import search_metric, search_outcome


@pytest.mark.parametrize(
    "change",
    [
        {"family": "injection", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
        {"family": "path", "path": "other.py", "function": "read", "function_lines": [100, 140]},
        {"family": "config_secrets", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
    ],
)
def test_matches_score_catches(change):
    change = {**change, "kind": "vulnerability", "files": [{"path": "app.py", "head_lines": [[20, 22]]}]}
    for path in ("app.py", "other.py"):
        for line in (9, 10, 22, 32, 33, 110, 500):
            for function in ("handler", "other"):
                for family in (change["family"], "unknown"):
                    finding = {"path": path, "line": line, "function": function, "family": family}
                    record = {**change, "push": {"admission": {"defects": [{**finding, "locations": [[path, line]]}]}}}
                    assert search_outcome(change, [finding]).hit == bool(catches(record))


def test_counts_scope_feedback_and_tolerance():
    change = {
        "kind": "vulnerability",
        "path": "app.py",
        "function": "handle",
        "family": "path",
        "function_lines": [20, 22],
        "files": [{"path": "app.py", "head_lines": [[20, 22]]}],
    }
    findings = [{"path": "app.py", "line": 10, "family": "path"}, {"path": "other.py", "line": 30, "family": "path"}]
    outcome = search_outcome(change, findings)
    assert (outcome.hit, outcome.suspicions, outcome.on_diff) == (True, 2, 1)
    assert "other.py:30" in outcome.feedback
    missed = search_outcome(change, findings, tolerance=0)
    assert not missed.hit
    assert "missed app.py:20 handle family path" in missed.feedback
    assert search_metric({"change": change}, {"findings": findings}) == 1.0
    assert search_metric(SimpleNamespace(change=change), SimpleNamespace(findings=[])) == 0.0


@pytest.mark.parametrize("count,expected", [(0, 1.0), (1, 0.8), (5, 0.0), (7, 0.0)])
def test_ordinary_metric(count, expected):
    assert search_metric({"kind": "ordinary", "files": []}, {"findings": [{}] * count}) == expected