mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(benchmark): valid detection benchmark spec + offline modules (groups 1-4)
Builds the benchmark from .kiro/specs/valid-detection-benchmark/brief.md:
sound labels from the fix (no SZZ), app-shaped verified positives,
ordinary negatives from the same repos, whole tool on unseen repos,
standard metrics.
Spec: requirements.md, design.md, tasks.md. Reuses the existing sourcing,
fix-site labelling, negative drawing, replay, and Wilson scoring; drops
the SZZ layer.
New offline modules under benchmarks/valid/:
- catalog.py positives labelled at the fix site (never introducing/last_touch)
- verify.py one LLM call per case (vulnerable? fix addresses it? reachable
in-repo?) + human spot-check; this is the app-shaped filter.
Honors the instrument rule: bytes read per function, reject on
an empty read.
- assemble.py >=20x ordinary negatives + a frozen, deterministic manifest with
the held-out guard
- metrics.py precision (PPV), false alarms per 100 pushes, exploit-verified
rate, and the plain headline, layered on score.py
36 new offline tests, 495 regression tests green. No network, no harvest,
no model calls in tests. Group 5 (the live pilot) is not started.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
3855945311
commit
43ef254134
@@ -0,0 +1,473 @@
|
||||
"""Offline fixtures for the valid detection benchmark."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
catalog = importlib.import_module("benchmarks.valid.catalog")
|
||||
verify = importlib.import_module("benchmarks.valid.verify")
|
||||
assemble = importlib.import_module("benchmarks.valid.assemble")
|
||||
metrics = importlib.import_module("benchmarks.valid.metrics")
|
||||
extract = importlib.import_module("benchmarks.unseen.extract")
|
||||
replay = importlib.import_module("benchmarks.unseen.replay")
|
||||
score = importlib.import_module("benchmarks.unseen.score")
|
||||
ChatResponse = importlib.import_module("openultrasast.tool_hunter").ChatResponse
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def offline(monkeypatch):
|
||||
def forbidden(*args, **kwargs):
|
||||
pytest.fail("network or SZZ was invoked")
|
||||
|
||||
import socket
|
||||
|
||||
monkeypatch.setattr(socket, "create_connection", forbidden)
|
||||
monkeypatch.setattr(socket.socket, "connect", forbidden)
|
||||
monkeypatch.setattr(extract, "introducing", forbidden)
|
||||
monkeypatch.setattr(extract.Git, "blame", forbidden)
|
||||
for var in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pair(tmp_path):
|
||||
commit_count = 0
|
||||
|
||||
def runner(argv, **kwargs):
|
||||
nonlocal commit_count
|
||||
assert not {"fetch", "clone", "pull", "push", "blame"} & set(argv)
|
||||
assert not any(arg.startswith("-L") for arg in argv)
|
||||
if "commit" in argv:
|
||||
timestamp = f"@{1600000000 + commit_count * 86400} +0000"
|
||||
kwargs["env"] = {**os.environ, "GIT_AUTHOR_DATE": timestamp, "GIT_COMMITTER_DATE": timestamp}
|
||||
commit_count += 1
|
||||
return subprocess.run(argv, **kwargs)
|
||||
|
||||
git = extract.Git(tmp_path, runner=runner)
|
||||
git.run("init", "-q")
|
||||
git.run("config", "user.name", "Fixture")
|
||||
git.run("config", "user.email", "[email protected]")
|
||||
pre = 'from flask import request\n@app.route("/run")\ndef handler():\n return eval(request.args["code"])\n'
|
||||
post = pre.replace('eval(request.args["code"])', 'int(request.args["code"])')
|
||||
commits = []
|
||||
for text in (pre, post):
|
||||
(tmp_path / "app.py").write_text(text)
|
||||
git.run("add", "app.py")
|
||||
git.run("commit", "-qm", "fixture")
|
||||
commits.append(git.run("rev-parse", "HEAD").strip())
|
||||
source = dict(
|
||||
repository="fixture/app",
|
||||
url="https://github.com/fixture/app",
|
||||
parent=commits[0],
|
||||
fix=commits[1],
|
||||
advisory="GHSA-fixture",
|
||||
cwe="CWE-94",
|
||||
family="injection",
|
||||
advisory_text="Request code is evaluated.",
|
||||
)
|
||||
(case,) = catalog.build_cases(git, source)
|
||||
return git, case
|
||||
|
||||
|
||||
def test_fix_site_catalog(pair, tmp_path):
|
||||
git, case = pair
|
||||
assert case["method"] == "fix_site"
|
||||
assert case["function"] == "handler"
|
||||
assert case["function_lines"] == [[3, 4]]
|
||||
assert case["language"] == "python" and case["cwe"] == "CWE-94"
|
||||
sides = catalog.read_functions(git, case)
|
||||
assert 'eval(request.args["code"])' in sides["pre_fix"]
|
||||
assert 'int(request.args["code"])' in sides["post_fix"]
|
||||
path = tmp_path / "catalog.toml"
|
||||
catalog.write_catalog(path, [case])
|
||||
assert tomllib.loads(path.read_text())["case"] == [case]
|
||||
|
||||
|
||||
class Stub:
|
||||
def __init__(self, **changes):
|
||||
self.calls = []
|
||||
self.answer = dict(
|
||||
vulnerable_pre_fix=True,
|
||||
fix_addresses_it=True,
|
||||
reachable_in_repo=True,
|
||||
entry_point="Flask route /run using request.args",
|
||||
confidence=0.86,
|
||||
advisory_cited=True,
|
||||
**changes,
|
||||
)
|
||||
|
||||
def complete(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
return ChatResponse(content=json.dumps(self.answer))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed", [None, "vulnerable_pre_fix", "fix_addresses_it", "reachable_in_repo"])
|
||||
@pytest.mark.parametrize("decision", ["accept", "reject", None])
|
||||
def test_verification_requires_three_checks_and_human(pair, capsys, failed, decision):
|
||||
git, case = pair
|
||||
stub = Stub()
|
||||
if failed:
|
||||
stub.answer[failed] = False
|
||||
record = verify.verify_case(git, case, client=stub)
|
||||
assert len(stub.calls) == 1
|
||||
assert record["verdict"] != "accepted"
|
||||
assert "pre_fix" in capsys.readouterr().out
|
||||
prompt = json.dumps(stub.calls[0]["messages"])
|
||||
assert "eval(" in prompt and "int(" in prompt and "@app.route" in prompt
|
||||
if decision:
|
||||
record = verify.spot_check(record, decision, "AB", "2026-10-12")
|
||||
assert (record["verdict"] == "accepted") == (not failed and decision == "accept")
|
||||
|
||||
|
||||
def test_empty_read_and_invalid_verdict(pair, monkeypatch):
|
||||
git, case = pair
|
||||
stub = Stub()
|
||||
monkeypatch.setattr(git, "blob", lambda *args: "")
|
||||
with pytest.raises(extract.InstrumentFailure):
|
||||
verify.verify_case(git, case, client=stub)
|
||||
assert not stub.calls
|
||||
|
||||
|
||||
def test_spot_check_cli(pair, tmp_path):
|
||||
git, case = pair
|
||||
record = verify.verify_case(git, case, client=Stub())
|
||||
path = tmp_path / "verification.json"
|
||||
verify.write_records(path, [record])
|
||||
assert verify.main(["spot-check", str(path), case["id"], "accept", "--reviewer", "AB", "--date", "2026-10-12"]) == 0
|
||||
assert json.loads(path.read_text())["cases"][0]["verdict"] == "accepted"
|
||||
|
||||
|
||||
def ordinary_rows(n=90):
|
||||
return [
|
||||
dict(
|
||||
head=f"{i + 10:040x}",
|
||||
parents=[f"{i + 9:040x}"],
|
||||
timestamp=i,
|
||||
author="Fixture",
|
||||
message="ordinary edit",
|
||||
files=[dict(path="other.py", added=1, removed=1, head_lines=[[2, 2]], functions=["other"], generated=False)],
|
||||
)
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
def accepted(case):
|
||||
return dict(
|
||||
id=case["id"],
|
||||
vulnerable_pre_fix=True,
|
||||
fix_addresses_it=True,
|
||||
reachable_in_repo=True,
|
||||
entry_point="Flask route /run",
|
||||
confidence=0.86,
|
||||
model="stub",
|
||||
advisory_cited=True,
|
||||
human_spot_check=dict(decision="accept", reviewer="AB", date="2026-10-12"),
|
||||
verdict="accepted",
|
||||
)
|
||||
|
||||
|
||||
def test_deterministic_freeze_and_replay(pair, tmp_path, monkeypatch):
|
||||
git, case = pair
|
||||
calls = []
|
||||
|
||||
def history(git, head):
|
||||
calls.append(head)
|
||||
return ordinary_rows()
|
||||
|
||||
monkeypatch.setattr(extract, "ordinary_history", history)
|
||||
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set(), seed=7)
|
||||
a = assemble.freeze(tmp_path / "a", **kwargs)
|
||||
b = assemble.freeze(tmp_path / "b", **kwargs)
|
||||
assert a == b and calls
|
||||
manifest = tomllib.loads((tmp_path / "a/pool-valid.toml").read_text())
|
||||
(row,) = manifest["repository"]
|
||||
pos = [c for c in row["changes"] if c["kind"] == "vulnerability"]
|
||||
neg = [c for c in row["changes"] if c["kind"] == "ordinary"]
|
||||
assert len(neg) >= 20 * len(pos) and len({c["head"] for c in neg}) == len(neg)
|
||||
assert pos[0]["base"] == case["commit"] and pos[0]["head"] == case["parent"]
|
||||
assert case["commit"] not in {c["head"] for c in neg}
|
||||
loaded = replay.load_slice(tmp_path / "a/pool-valid.toml", 1)
|
||||
assert len(loaded) == len(row["changes"]) and loaded[0]["case_id"] == case["id"]
|
||||
assert assemble.freeze(tmp_path / "a", **kwargs) == a
|
||||
|
||||
|
||||
def test_held_out_guard_drops_used_repo_before_reading(pair, tmp_path, monkeypatch):
|
||||
git, case = pair
|
||||
other = {**case, "repo": "fixture/used", "id": "used-case"}
|
||||
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows())
|
||||
result = assemble.freeze(
|
||||
tmp_path,
|
||||
cases=[case, other],
|
||||
verifications=[accepted(case), accepted(other)],
|
||||
repos={case["repo"]: git},
|
||||
used={"https://github.com/FIXTURE/USED.git"},
|
||||
seed=7,
|
||||
)
|
||||
assert result["guard_result"] == 0 and result["dropped_count"] == 1
|
||||
assert [r["repository"] for r in tomllib.loads((tmp_path / "pool-valid.toml").read_text())["repository"]] == [case["repo"]]
|
||||
|
||||
|
||||
def record(id, kind, flagged, caught=True):
|
||||
return dict(
|
||||
id=id,
|
||||
case_id=id,
|
||||
repo="fixture/app",
|
||||
kind=kind,
|
||||
family="injection",
|
||||
path="app.py",
|
||||
function="handler",
|
||||
files=[dict(path="app.py", head_lines=[[4, 4]])],
|
||||
instrument=dict(head_verified=True, base_verified=True, hook_exit=0, changed_bytes={"app.py": 40}, hook_bytes_read=40),
|
||||
push=dict(admission=dict(defects=[dict(locations=[["app.py", 4]], family="injection" if caught else "path")] if flagged else [])),
|
||||
)
|
||||
|
||||
|
||||
def hand_wilson(k, n):
|
||||
denominator = n + 1.96**2
|
||||
center = (k + 1.96**2 / 2) / denominator
|
||||
half = 1.96 * math.sqrt(k * (n - k) / n + 1.96**2 / 4) / denominator
|
||||
return [center - half, center + half]
|
||||
|
||||
|
||||
def proof(id, observed=True):
|
||||
return dict(
|
||||
case_id=id,
|
||||
verification=dict(
|
||||
outcome="demonstrated" if observed else "inconclusive",
|
||||
sides=[dict(outcome="observed", runs=[dict(observed=v)] * 3) for v in (observed, False)],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_hand_calculations():
|
||||
rows = [
|
||||
record("p1", "vulnerability", True),
|
||||
record("p2", "vulnerability", True),
|
||||
record("p3", "vulnerability", False),
|
||||
record("n1", "ordinary", True),
|
||||
record("n2", "ordinary", False),
|
||||
]
|
||||
base = score.score(rows)
|
||||
result = metrics.summarize(rows, scored=base, proofs=[proof("p1")])
|
||||
assert result["recall"]["rate"] == pytest.approx(2 / 3)
|
||||
assert result["recall"]["wilson95"] == pytest.approx(hand_wilson(2, 3))
|
||||
assert result["recall"]["cluster95"] == pytest.approx([2 / 3, 2 / 3])
|
||||
assert result["precision"]["rate"] == pytest.approx(2 / 3)
|
||||
assert result["precision"]["wilson95"] == pytest.approx(hand_wilson(2, 3))
|
||||
fa = result["false_alarms_per_100_pushes"]
|
||||
assert fa["rate"] == 50 and fa["wilson95"] == pytest.approx([v * 100 for v in hand_wilson(1, 2)])
|
||||
assert fa["cluster95"] == [50, 50]
|
||||
proved = result["exploit_verified"]
|
||||
assert proved["count"] == proved["n"] == 1 and proved["rate"] == 1
|
||||
assert proved["flagged_positives"] == 2 and proved["not_yet_proved"] == ["p2"]
|
||||
assert proved["wilson95"] == pytest.approx(hand_wilson(1, 1))
|
||||
assert result["headline"] == (
|
||||
"On unseen app repos: recall 66.7% (Wilson 20.8-93.9), false alarms 50.0 per 100 pushes, "
|
||||
"precision 66.7%; exploit-verified 1 of 2 (1 not yet proved)."
|
||||
)
|
||||
|
||||
|
||||
def test_precision_counts_unmatched_positive_flags_and_proof_failures():
|
||||
rows = [record("p", "vulnerability", True, caught=False), record("q", "vulnerability", True), record("n", "ordinary", True)]
|
||||
result = metrics.summarize(rows, proofs=[proof("q", observed=False)])
|
||||
assert result["precision"]["rate"] == pytest.approx(1 / 3)
|
||||
assert result["exploit_verified"]["count"] == 0
|
||||
assert result["exploit_verified"]["n"] == 1
|
||||
assert result["exploit_verified"]["not_yet_proved"] == ["p"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix,language", [(".js", "javascript"), (".ts", "typescript"), (".php", None)])
|
||||
def test_catalog_language_scope_and_single_line_function(suffix, language):
|
||||
class Git:
|
||||
def parents(self, sha):
|
||||
return ["a" * 40]
|
||||
|
||||
def stats(self, *args):
|
||||
return [("app" + suffix, 1, 1)]
|
||||
|
||||
def hunks(self, *args):
|
||||
return [(1, 1, 1, 1)]
|
||||
|
||||
def blob(self, sha, path):
|
||||
return "function handler(value) { return " + ("eval(value)" if sha == "a" * 40 else "Number(value)") + "; }"
|
||||
|
||||
cases = catalog.build_cases(
|
||||
Git(),
|
||||
dict(
|
||||
repository="fixture/app",
|
||||
url="https://github.com/fixture/app",
|
||||
fix="b" * 40,
|
||||
parent="a" * 40,
|
||||
advisory="GHSA-fixture",
|
||||
cwe="CWE-94",
|
||||
family="injection",
|
||||
),
|
||||
)
|
||||
if language:
|
||||
(case,) = cases
|
||||
assert case["language"] == language and "eval(value)" in case["pre_fix"]
|
||||
else:
|
||||
assert cases == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty", [False, True])
|
||||
def test_shipped_provider_is_bounded_to_one_attempt(pair, monkeypatch, empty):
|
||||
from openultrasast.model.endpoint import ChatEndpoint, DeepSeekChatClient
|
||||
from openultrasast.provider.openrouter import OpenRouterChatClient
|
||||
|
||||
git, case = pair
|
||||
calls = []
|
||||
|
||||
def transport(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
assert self.max_attempts == 1
|
||||
assert kwargs["extra_body"]["max_tokens"] == 1200
|
||||
assert kwargs["extra_body"]["thinking"] == {"type": "disabled"}
|
||||
return {"choices": [{"message": {"content": "" if empty else json.dumps(Stub().answer)}}]}
|
||||
|
||||
endpoint = ChatEndpoint(provider="deepseek", base_url="", thinking=False)
|
||||
client = DeepSeekChatClient(OpenRouterChatClient(api_key="offline-fixture"), endpoint=endpoint)
|
||||
monkeypatch.setattr(OpenRouterChatClient, "complete_chat_raw", transport)
|
||||
monkeypatch.setattr(verify, "resolve_chat_endpoint", lambda config: (client, endpoint))
|
||||
if empty:
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
verify.verify_case(git, case)
|
||||
else:
|
||||
assert verify.verify_case(git, case)["verdict"] == "pending_human"
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,value",
|
||||
[("reachable_in_repo", "true"), ("confidence", True), ("confidence", 1.5), ("confidence", float("nan")), ("entry_point", None)],
|
||||
)
|
||||
def test_malformed_model_reply_cannot_accept(pair, key, value):
|
||||
git, case = pair
|
||||
stub = Stub()
|
||||
stub.answer[key] = value
|
||||
with pytest.raises(ValueError):
|
||||
verify.verify_case(git, case, client=stub)
|
||||
assert len(stub.calls) == 1
|
||||
|
||||
|
||||
def test_empty_entry_point_cannot_accept(pair):
|
||||
git, case = pair
|
||||
stub = Stub()
|
||||
stub.answer["entry_point"] = ""
|
||||
assert verify.spot_check(verify.verify_case(git, case, client=stub), "accept", "AB", "2026-10-12")["verdict"] == "rejected"
|
||||
|
||||
|
||||
def test_real_local_history_freezes_without_network(pair, tmp_path):
|
||||
git, case = pair
|
||||
for i in range(55):
|
||||
(git.path / "other.py").write_text(f"def other():\n return {i}\n")
|
||||
git.run("add", "other.py")
|
||||
git.run("commit", "-qm", "ordinary edit")
|
||||
result = assemble.freeze(tmp_path / "freeze", cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
||||
assert result["label_check_counts"]["confirmed"] == 1
|
||||
assert len(replay.load_slice(tmp_path / "freeze/pool-valid.toml", 1)) == 21
|
||||
|
||||
|
||||
def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):
|
||||
git, case = pair
|
||||
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows(19))
|
||||
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
||||
with pytest.raises(ValueError, match="insufficient ordinary"):
|
||||
assemble.freeze(tmp_path, **kwargs)
|
||||
with pytest.raises(ValueError, match="at least 20"):
|
||||
assemble.freeze(tmp_path, ratio=19, **kwargs)
|
||||
assert not (tmp_path / "freeze-valid.json").exists()
|
||||
|
||||
|
||||
def test_freeze_cannot_overwrite_or_trust_verdict_alone(pair, tmp_path, monkeypatch):
|
||||
git, case = pair
|
||||
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows())
|
||||
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
||||
assemble.freeze(tmp_path, **kwargs)
|
||||
before = (tmp_path / "pool-valid.toml").read_bytes()
|
||||
with pytest.raises(FileExistsError):
|
||||
assemble.freeze(tmp_path, seed=1, **kwargs)
|
||||
assert (tmp_path / "pool-valid.toml").read_bytes() == before
|
||||
kwargs["verifications"][0]["human_spot_check"]["decision"] = "reject"
|
||||
with pytest.raises(ValueError, match="no accepted"):
|
||||
assemble.freeze(tmp_path / "rejected", **kwargs)
|
||||
|
||||
|
||||
def test_no_flags_and_invalid_proof_are_not_false_proofs():
|
||||
result = metrics.summarize([record("p", "vulnerability", False), record("n", "ordinary", False)])
|
||||
assert result["precision"] == dict(n=0, count=0, rate=None, wilson95=[None, None])
|
||||
assert result["exploit_verified"]["rate"] is None
|
||||
bad = proof("p")
|
||||
bad["verification"]["sides"][1]["runs"] = [dict(observed=True)] * 3
|
||||
result = metrics.summarize([record("p", "vulnerability", True)], proofs=[bad])
|
||||
assert result["exploit_verified"]["count"] == 0
|
||||
|
||||
|
||||
def test_multi_function_fix_is_one_push_with_all_case_ids(pair, tmp_path, monkeypatch):
|
||||
git, original = pair
|
||||
parent_text = "def first(value):\n return eval(value)\n\ndef second(value):\n return eval(value)\n"
|
||||
commits = []
|
||||
for text in (parent_text, parent_text.replace("eval(value)", "int(value)")):
|
||||
(git.path / "multi.py").write_text(text)
|
||||
git.run("add", "multi.py")
|
||||
git.run("commit", "-qm", "fixture")
|
||||
commits.append(git.run("rev-parse", "HEAD").strip())
|
||||
source = dict(
|
||||
repository=original["repo"],
|
||||
url=original["url"],
|
||||
parent=commits[0],
|
||||
fix=commits[1],
|
||||
advisory="GHSA-multi",
|
||||
cwe="CWE-94",
|
||||
family="injection",
|
||||
advisory_text="Two functions evaluate user code.",
|
||||
)
|
||||
cases = catalog.build_cases(git, source)
|
||||
assert len(cases) == 2
|
||||
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows())
|
||||
assemble.freeze(
|
||||
tmp_path / "multi", cases=cases, verifications=[accepted(case) for case in cases], repos={original["repo"]: git}, used=set()
|
||||
)
|
||||
loaded = replay.load_slice(tmp_path / "multi/pool-valid.toml", 1)
|
||||
assert len(loaded) == 41 and len({row["id"] for row in loaded}) == 41
|
||||
(positive,) = [row for row in loaded if row["kind"] == "vulnerability"]
|
||||
assert set(positive["case_ids"]) == {case["id"] for case in cases}
|
||||
target = positive["targets"][1]
|
||||
detected = record(positive["id"], "vulnerability", True)
|
||||
detected.update({key: value for key, value in positive.items() if key not in {"id", "repo"}})
|
||||
detected["push"] = dict(
|
||||
admission=dict(
|
||||
defects=[dict(locations=[[target["path"], target["function_lines"][0]]], function=target["function"], family="injection")]
|
||||
)
|
||||
)
|
||||
detected["instrument"]["changed_bytes"] = {target["path"]: 40}
|
||||
result = metrics.summarize([detected, record("negative", "ordinary", False)], proofs=[proof(positive["case_ids"][0])])
|
||||
assert result["recall"]["n"] == result["recall"]["count"] == 1
|
||||
assert result["precision"]["rate"] == 1
|
||||
assert result["exploit_verified"]["flagged_positives"] == 2
|
||||
assert result["exploit_verified"]["flagged_positive_pushes"] == 1
|
||||
assert result["exploit_verified"]["not_yet_proved"] == [positive["case_ids"][1]]
|
||||
|
||||
|
||||
def test_grouped_targets_different_families_use_shipped_scorer():
|
||||
row = record("p", "vulnerability", True)
|
||||
row["targets"] = [
|
||||
dict(case_id="one", path="miss.py", function="first", function_lines=[40, 42], family="path"),
|
||||
dict(case_id="two", path="app.py", function="handler", function_lines=[3, 4], family="injection"),
|
||||
]
|
||||
row["case_ids"] = ["one", "two"]
|
||||
row.update({key: value for key, value in row["targets"][0].items() if key != "case_id"})
|
||||
row["files"] = [{"path": "miss.py", "head_lines": [[40, 42]]}]
|
||||
legacy = score.score([row])
|
||||
assert legacy["catch"]["count"] == 0
|
||||
result = metrics.summarize([row], scored=legacy)
|
||||
assert result["recall"]["count"] == 1 and result["precision"]["rate"] == 1
|
||||
Reference in New Issue
Block a user