mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5.5
2b65680d5d
Sourced apart from the 12 pilot repositories: 212 candidates (pool-fresh, pool-retry, OSV for PyPI, npm, Maven and Packagist), 49 dropped because the benchmarks already name their repository, 413 catalog locations, 99 passing all three verifier checks in 28 repositories. Short of the 40-positive target. catalog.py now reads .java and .php. Freeze waits on the maintainer's spot-check (CANDIDATES.md, SOURCING.md). Co-Authored-By: Claude Opus 5.5 <[email protected]>
476 lines
19 KiB
Python
476 lines
19 KiB
Python
"""Offline fixtures for the valid detection benchmark."""
|
|
|
|
import importlib
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tomllib
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
catalog = importlib.import_module("benchmarks.valid.catalog")
|
|
verify = importlib.import_module("benchmarks.valid.verify")
|
|
assemble = importlib.import_module("benchmarks.valid.assemble")
|
|
metrics = importlib.import_module("benchmarks.valid.metrics")
|
|
extract = importlib.import_module("benchmarks.unseen.extract")
|
|
replay = importlib.import_module("benchmarks.unseen.replay")
|
|
score = importlib.import_module("benchmarks.unseen.score")
|
|
ChatResponse = importlib.import_module("openultrasast.tool_hunter").ChatResponse
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def offline(monkeypatch):
|
|
def forbidden(*args, **kwargs):
|
|
pytest.fail("network or SZZ was invoked")
|
|
|
|
import socket
|
|
|
|
monkeypatch.setattr(socket, "create_connection", forbidden)
|
|
monkeypatch.setattr(socket.socket, "connect", forbidden)
|
|
monkeypatch.setattr(extract, "introducing", forbidden)
|
|
monkeypatch.setattr(extract.Git, "blame", forbidden)
|
|
for var in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"):
|
|
monkeypatch.delenv(var, raising=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def pair(tmp_path):
|
|
commit_count = 0
|
|
|
|
def runner(argv, **kwargs):
|
|
nonlocal commit_count
|
|
assert not {"fetch", "clone", "pull", "push", "blame"} & set(argv)
|
|
assert not any(arg.startswith("-L") for arg in argv)
|
|
if "commit" in argv:
|
|
timestamp = f"@{1600000000 + commit_count * 86400} +0000"
|
|
kwargs["env"] = {**os.environ, "GIT_AUTHOR_DATE": timestamp, "GIT_COMMITTER_DATE": timestamp}
|
|
commit_count += 1
|
|
return subprocess.run(argv, **kwargs)
|
|
|
|
git = extract.Git(tmp_path, runner=runner)
|
|
git.run("init", "-q")
|
|
git.run("config", "user.name", "Fixture")
|
|
git.run("config", "user.email", "[email protected]")
|
|
pre = 'from flask import request\n@app.route("/run")\ndef handler():\n return eval(request.args["code"])\n'
|
|
post = pre.replace('eval(request.args["code"])', 'int(request.args["code"])')
|
|
commits = []
|
|
for text in (pre, post):
|
|
(tmp_path / "app.py").write_text(text)
|
|
git.run("add", "app.py")
|
|
git.run("commit", "-qm", "fixture")
|
|
commits.append(git.run("rev-parse", "HEAD").strip())
|
|
source = dict(
|
|
repository="fixture/app",
|
|
url="https://github.com/fixture/app",
|
|
parent=commits[0],
|
|
fix=commits[1],
|
|
advisory="GHSA-fixture",
|
|
cwe="CWE-94",
|
|
family="injection",
|
|
advisory_text="Request code is evaluated.",
|
|
)
|
|
(case,) = catalog.build_cases(git, source)
|
|
return git, case
|
|
|
|
|
|
def test_fix_site_catalog(pair, tmp_path):
|
|
git, case = pair
|
|
assert case["method"] == "fix_site"
|
|
assert case["function"] == "handler"
|
|
assert case["function_lines"] == [[3, 4]]
|
|
assert case["language"] == "python" and case["cwe"] == "CWE-94"
|
|
sides = catalog.read_functions(git, case)
|
|
assert 'eval(request.args["code"])' in sides["pre_fix"]
|
|
assert 'int(request.args["code"])' in sides["post_fix"]
|
|
path = tmp_path / "catalog.toml"
|
|
catalog.write_catalog(path, [case])
|
|
assert tomllib.loads(path.read_text())["case"] == [case]
|
|
|
|
|
|
class Stub:
|
|
def __init__(self, **changes):
|
|
self.calls = []
|
|
self.answer = dict(
|
|
vulnerable_pre_fix=True,
|
|
fix_addresses_it=True,
|
|
reachable_in_repo=True,
|
|
entry_point="Flask route /run using request.args",
|
|
confidence=0.86,
|
|
advisory_cited=True,
|
|
**changes,
|
|
)
|
|
|
|
def complete(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
return ChatResponse(content=json.dumps(self.answer))
|
|
|
|
|
|
@pytest.mark.parametrize("failed", [None, "vulnerable_pre_fix", "fix_addresses_it", "reachable_in_repo"])
|
|
@pytest.mark.parametrize("decision", ["accept", "reject", None])
|
|
def test_verification_requires_three_checks_and_human(pair, capsys, failed, decision):
|
|
git, case = pair
|
|
stub = Stub()
|
|
if failed:
|
|
stub.answer[failed] = False
|
|
record = verify.verify_case(git, case, client=stub)
|
|
assert len(stub.calls) == 1
|
|
assert record["verdict"] != "accepted"
|
|
assert "pre_fix" in capsys.readouterr().out
|
|
prompt = json.dumps(stub.calls[0]["messages"])
|
|
assert "eval(" in prompt and "int(" in prompt and "@app.route" in prompt
|
|
if decision:
|
|
record = verify.spot_check(record, decision, "AB", "2026-10-12")
|
|
assert (record["verdict"] == "accepted") == (not failed and decision == "accept")
|
|
|
|
|
|
def test_empty_read_and_invalid_verdict(pair, monkeypatch):
|
|
git, case = pair
|
|
stub = Stub()
|
|
monkeypatch.setattr(git, "blob", lambda *args: "")
|
|
with pytest.raises(extract.InstrumentFailure):
|
|
verify.verify_case(git, case, client=stub)
|
|
assert not stub.calls
|
|
|
|
|
|
def test_spot_check_cli(pair, tmp_path):
|
|
git, case = pair
|
|
record = verify.verify_case(git, case, client=Stub())
|
|
path = tmp_path / "verification.json"
|
|
verify.write_records(path, [record])
|
|
assert verify.main(["spot-check", str(path), case["id"], "accept", "--reviewer", "AB", "--date", "2026-10-12"]) == 0
|
|
assert json.loads(path.read_text())["cases"][0]["verdict"] == "accepted"
|
|
|
|
|
|
def ordinary_rows(n=90):
|
|
return [
|
|
dict(
|
|
head=f"{i + 10:040x}",
|
|
parents=[f"{i + 9:040x}"],
|
|
timestamp=i,
|
|
author="Fixture",
|
|
message="ordinary edit",
|
|
files=[dict(path="other.py", added=1, removed=1, head_lines=[[2, 2]], functions=["other"], generated=False)],
|
|
)
|
|
for i in range(n)
|
|
]
|
|
|
|
|
|
def accepted(case):
|
|
return dict(
|
|
id=case["id"],
|
|
vulnerable_pre_fix=True,
|
|
fix_addresses_it=True,
|
|
reachable_in_repo=True,
|
|
entry_point="Flask route /run",
|
|
confidence=0.86,
|
|
model="stub",
|
|
advisory_cited=True,
|
|
human_spot_check=dict(decision="accept", reviewer="AB", date="2026-10-12"),
|
|
verdict="accepted",
|
|
)
|
|
|
|
|
|
def test_deterministic_freeze_and_replay(pair, tmp_path, monkeypatch):
|
|
git, case = pair
|
|
calls = []
|
|
|
|
def history(git, head):
|
|
calls.append(head)
|
|
return ordinary_rows()
|
|
|
|
monkeypatch.setattr(extract, "ordinary_history", history)
|
|
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set(), seed=7)
|
|
a = assemble.freeze(tmp_path / "a", **kwargs)
|
|
b = assemble.freeze(tmp_path / "b", **kwargs)
|
|
assert a == b and calls
|
|
manifest = tomllib.loads((tmp_path / "a/pool-valid.toml").read_text())
|
|
(row,) = manifest["repository"]
|
|
pos = [c for c in row["changes"] if c["kind"] == "vulnerability"]
|
|
neg = [c for c in row["changes"] if c["kind"] == "ordinary"]
|
|
assert len(neg) >= 20 * len(pos) and len({c["head"] for c in neg}) == len(neg)
|
|
assert pos[0]["base"] == case["commit"] and pos[0]["head"] == case["parent"]
|
|
assert case["commit"] not in {c["head"] for c in neg}
|
|
loaded = replay.load_slice(tmp_path / "a/pool-valid.toml", 1)
|
|
assert len(loaded) == len(row["changes"]) and loaded[0]["case_id"] == case["id"]
|
|
assert assemble.freeze(tmp_path / "a", **kwargs) == a
|
|
|
|
|
|
def test_held_out_guard_drops_used_repo_before_reading(pair, tmp_path, monkeypatch):
|
|
git, case = pair
|
|
other = {**case, "repo": "fixture/used", "id": "used-case"}
|
|
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows())
|
|
result = assemble.freeze(
|
|
tmp_path,
|
|
cases=[case, other],
|
|
verifications=[accepted(case), accepted(other)],
|
|
repos={case["repo"]: git},
|
|
used={"https://github.com/FIXTURE/USED.git"},
|
|
seed=7,
|
|
)
|
|
assert result["guard_result"] == 0 and result["dropped_count"] == 1
|
|
assert [r["repository"] for r in tomllib.loads((tmp_path / "pool-valid.toml").read_text())["repository"]] == [case["repo"]]
|
|
|
|
|
|
def record(id, kind, flagged, caught=True):
|
|
return dict(
|
|
id=id,
|
|
case_id=id,
|
|
repo="fixture/app",
|
|
kind=kind,
|
|
family="injection",
|
|
path="app.py",
|
|
function="handler",
|
|
files=[dict(path="app.py", head_lines=[[4, 4]])],
|
|
instrument=dict(head_verified=True, base_verified=True, hook_exit=0, changed_bytes={"app.py": 40}, hook_bytes_read=40),
|
|
push=dict(admission=dict(defects=[dict(locations=[["app.py", 4]], family="injection" if caught else "path")] if flagged else [])),
|
|
)
|
|
|
|
|
|
def hand_wilson(k, n):
|
|
denominator = n + 1.96**2
|
|
center = (k + 1.96**2 / 2) / denominator
|
|
half = 1.96 * math.sqrt(k * (n - k) / n + 1.96**2 / 4) / denominator
|
|
return [center - half, center + half]
|
|
|
|
|
|
def proof(id, observed=True):
|
|
return dict(
|
|
case_id=id,
|
|
verification=dict(
|
|
outcome="demonstrated" if observed else "inconclusive",
|
|
sides=[dict(outcome="observed", runs=[dict(observed=v)] * 3) for v in (observed, False)],
|
|
),
|
|
)
|
|
|
|
|
|
def test_metrics_hand_calculations():
|
|
rows = [
|
|
record("p1", "vulnerability", True),
|
|
record("p2", "vulnerability", True),
|
|
record("p3", "vulnerability", False),
|
|
record("n1", "ordinary", True),
|
|
record("n2", "ordinary", False),
|
|
]
|
|
base = score.score(rows)
|
|
result = metrics.summarize(rows, scored=base, proofs=[proof("p1")])
|
|
assert result["recall"]["rate"] == pytest.approx(2 / 3)
|
|
assert result["recall"]["wilson95"] == pytest.approx(hand_wilson(2, 3))
|
|
assert result["recall"]["cluster95"] == pytest.approx([2 / 3, 2 / 3])
|
|
assert result["precision"]["rate"] == pytest.approx(2 / 3)
|
|
assert result["precision"]["wilson95"] == pytest.approx(hand_wilson(2, 3))
|
|
fa = result["false_alarms_per_100_pushes"]
|
|
assert fa["rate"] == 50 and fa["wilson95"] == pytest.approx([v * 100 for v in hand_wilson(1, 2)])
|
|
assert fa["cluster95"] == [50, 50]
|
|
proved = result["exploit_verified"]
|
|
assert proved["count"] == proved["n"] == 1 and proved["rate"] == 1
|
|
assert proved["flagged_positives"] == 2 and proved["not_yet_proved"] == ["p2"]
|
|
assert proved["wilson95"] == pytest.approx(hand_wilson(1, 1))
|
|
assert result["headline"] == (
|
|
"On unseen app repos: recall 66.7% (Wilson 20.8-93.9), false alarms 50.0 per 100 pushes, "
|
|
"precision 66.7%; exploit-verified 1 of 2 (1 not yet proved)."
|
|
)
|
|
|
|
|
|
def test_precision_counts_unmatched_positive_flags_and_proof_failures():
|
|
rows = [record("p", "vulnerability", True, caught=False), record("q", "vulnerability", True), record("n", "ordinary", True)]
|
|
result = metrics.summarize(rows, proofs=[proof("q", observed=False)])
|
|
assert result["precision"]["rate"] == pytest.approx(1 / 3)
|
|
assert result["exploit_verified"]["count"] == 0
|
|
assert result["exploit_verified"]["n"] == 1
|
|
assert result["exploit_verified"]["not_yet_proved"] == ["p"]
|
|
|
|
|
|
@pytest.mark.parametrize("suffix,language", [(".js", "javascript"), (".ts", "typescript"), (".php", "php"), (".rb", None)])
|
|
def test_catalog_language_scope_and_single_line_function(suffix, language):
|
|
class Git:
|
|
def parents(self, sha):
|
|
return ["a" * 40]
|
|
|
|
def stats(self, *args):
|
|
return [("app" + suffix, 1, 1)]
|
|
|
|
def hunks(self, *args):
|
|
return [(1, 1, 1, 1)]
|
|
|
|
def blob(self, sha, path):
|
|
return "function handler(value) { return " + ("eval(value)" if sha == "a" * 40 else "Number(value)") + "; }"
|
|
|
|
cases = catalog.build_cases(
|
|
Git(),
|
|
dict(
|
|
repository="fixture/app",
|
|
url="https://github.com/fixture/app",
|
|
fix="b" * 40,
|
|
parent="a" * 40,
|
|
advisory="GHSA-fixture",
|
|
cwe="CWE-94",
|
|
family="injection",
|
|
),
|
|
)
|
|
if language:
|
|
(case,) = cases
|
|
assert case["language"] == language and "eval(value)" in case["pre_fix"]
|
|
else:
|
|
assert cases == []
|
|
|
|
|
|
@pytest.mark.parametrize("empty", [False, True])
|
|
def test_shipped_provider_is_bounded_to_one_attempt(pair, monkeypatch, empty):
|
|
from openultrasast.model.endpoint import ChatEndpoint, DeepSeekChatClient
|
|
from openultrasast.provider.openrouter import OpenRouterChatClient
|
|
|
|
git, case = pair
|
|
calls = []
|
|
|
|
def transport(self, **kwargs):
|
|
calls.append(kwargs)
|
|
assert self.max_attempts == 1
|
|
assert kwargs["extra_body"]["max_tokens"] == 1200
|
|
assert kwargs["extra_body"]["thinking"] == {"type": "disabled"}
|
|
return {"choices": [{"message": {"content": "" if empty else json.dumps(Stub().answer)}}]}
|
|
|
|
endpoint = ChatEndpoint(provider="deepseek", base_url="", thinking=False)
|
|
client = DeepSeekChatClient(OpenRouterChatClient(api_key="offline-fixture"), endpoint=endpoint)
|
|
monkeypatch.setattr(OpenRouterChatClient, "complete_chat_raw", transport)
|
|
monkeypatch.setattr(verify, "resolve_chat_endpoint", lambda config: (client, endpoint))
|
|
if empty:
|
|
with pytest.raises(json.JSONDecodeError):
|
|
verify.verify_case(git, case)
|
|
else:
|
|
assert verify.verify_case(git, case)["verdict"] == "pending_human"
|
|
assert len(calls) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"key,value",
|
|
[("reachable_in_repo", "true"), ("confidence", True), ("confidence", 1.5), ("confidence", float("nan")), ("entry_point", None)],
|
|
)
|
|
def test_malformed_model_reply_cannot_accept(pair, key, value):
|
|
git, case = pair
|
|
stub = Stub()
|
|
stub.answer[key] = value
|
|
with pytest.raises(ValueError):
|
|
verify.verify_case(git, case, client=stub)
|
|
assert len(stub.calls) == 1
|
|
|
|
|
|
def test_empty_entry_point_cannot_accept(pair):
|
|
git, case = pair
|
|
stub = Stub()
|
|
stub.answer["entry_point"] = ""
|
|
assert verify.spot_check(verify.verify_case(git, case, client=stub), "accept", "AB", "2026-10-12")["verdict"] == "rejected"
|
|
|
|
|
|
def test_real_local_history_freezes_without_network(pair, tmp_path):
|
|
git, case = pair
|
|
for i in range(55):
|
|
(git.path / "other.py").write_text(f"def other():\n return {i}\n")
|
|
git.run("add", "other.py")
|
|
git.run("commit", "-qm", "ordinary edit")
|
|
result = assemble.freeze(tmp_path / "freeze", cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
|
assert result["label_check_counts"]["confirmed"] == 1
|
|
assert len(replay.load_slice(tmp_path / "freeze/pool-valid.toml", 1)) == 21
|
|
|
|
|
|
def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):
|
|
git, case = pair
|
|
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows(19))
|
|
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
|
with pytest.raises(ValueError, match="insufficient ordinary"):
|
|
assemble.freeze(tmp_path, **kwargs)
|
|
with pytest.raises(ValueError, match="at least 20"):
|
|
assemble.freeze(tmp_path, ratio=19, **kwargs)
|
|
assert not (tmp_path / "freeze-valid.json").exists()
|
|
|
|
|
|
def test_freeze_cannot_overwrite_or_trust_verdict_alone(pair, tmp_path, monkeypatch):
|
|
git, case = pair
|
|
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows())
|
|
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
|
assemble.freeze(tmp_path, **kwargs)
|
|
before = (tmp_path / "pool-valid.toml").read_bytes()
|
|
with pytest.raises(FileExistsError):
|
|
assemble.freeze(tmp_path, seed=1, **kwargs)
|
|
assert (tmp_path / "pool-valid.toml").read_bytes() == before
|
|
kwargs["verifications"][0]["human_spot_check"]["decision"] = "reject"
|
|
with pytest.raises(ValueError, match="no accepted"):
|
|
assemble.freeze(tmp_path / "rejected", **kwargs)
|
|
|
|
|
|
def test_no_flags_and_invalid_proof_are_not_false_proofs():
|
|
result = metrics.summarize([record("p", "vulnerability", False), record("n", "ordinary", False)])
|
|
assert result["precision"] == dict(n=0, count=0, rate=None, wilson95=[None, None])
|
|
assert result["exploit_verified"]["rate"] is None
|
|
bad = proof("p")
|
|
bad["verification"]["sides"][1]["runs"] = [dict(observed=True)] * 3
|
|
result = metrics.summarize([record("p", "vulnerability", True)], proofs=[bad])
|
|
assert result["exploit_verified"]["count"] == 0
|
|
|
|
|
|
def test_multi_function_fix_is_one_push_with_all_case_ids(pair, tmp_path, monkeypatch):
|
|
git, original = pair
|
|
parent_text = "def first(value):\n return eval(value)\n\ndef second(value):\n return eval(value)\n"
|
|
commits = []
|
|
for text in (parent_text, parent_text.replace("eval(value)", "int(value)")):
|
|
(git.path / "multi.py").write_text(text)
|
|
git.run("add", "multi.py")
|
|
git.run("commit", "-qm", "fixture")
|
|
commits.append(git.run("rev-parse", "HEAD").strip())
|
|
source = dict(
|
|
repository=original["repo"],
|
|
url=original["url"],
|
|
parent=commits[0],
|
|
fix=commits[1],
|
|
advisory="GHSA-multi",
|
|
cwe="CWE-94",
|
|
family="injection",
|
|
advisory_text="Two functions evaluate user code.",
|
|
)
|
|
cases = catalog.build_cases(git, source)
|
|
assert len(cases) == 2
|
|
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows())
|
|
assemble.freeze(
|
|
tmp_path / "multi", cases=cases, verifications=[accepted(case) for case in cases], repos={original["repo"]: git}, used=set()
|
|
)
|
|
loaded = replay.load_slice(tmp_path / "multi/pool-valid.toml", 1)
|
|
# One fix commit is one positive change, so the 20x negative ratio is 20, not 20 per
|
|
# accepted function. Two accepted functions still collapse to a single vulnerability push.
|
|
assert len(loaded) == 21 and len({row["id"] for row in loaded}) == 21
|
|
(positive,) = [row for row in loaded if row["kind"] == "vulnerability"]
|
|
assert set(positive["case_ids"]) == {case["id"] for case in cases}
|
|
target = positive["targets"][1]
|
|
detected = record(positive["id"], "vulnerability", True)
|
|
detected.update({key: value for key, value in positive.items() if key not in {"id", "repo"}})
|
|
detected["push"] = dict(
|
|
admission=dict(
|
|
defects=[dict(locations=[[target["path"], target["function_lines"][0]]], function=target["function"], family="injection")]
|
|
)
|
|
)
|
|
detected["instrument"]["changed_bytes"] = {target["path"]: 40}
|
|
result = metrics.summarize([detected, record("negative", "ordinary", False)], proofs=[proof(positive["case_ids"][0])])
|
|
assert result["recall"]["n"] == result["recall"]["count"] == 1
|
|
assert result["precision"]["rate"] == 1
|
|
assert result["exploit_verified"]["flagged_positives"] == 2
|
|
assert result["exploit_verified"]["flagged_positive_pushes"] == 1
|
|
assert result["exploit_verified"]["not_yet_proved"] == [positive["case_ids"][1]]
|
|
|
|
|
|
def test_grouped_targets_different_families_use_shipped_scorer():
|
|
row = record("p", "vulnerability", True)
|
|
row["targets"] = [
|
|
dict(case_id="one", path="miss.py", function="first", function_lines=[40, 42], family="path"),
|
|
dict(case_id="two", path="app.py", function="handler", function_lines=[3, 4], family="injection"),
|
|
]
|
|
row["case_ids"] = ["one", "two"]
|
|
row.update({key: value for key, value in row["targets"][0].items() if key != "case_id"})
|
|
row["files"] = [{"path": "miss.py", "head_lines": [[40, 42]]}]
|
|
legacy = score.score([row])
|
|
assert legacy["catch"]["count"] == 0
|
|
result = metrics.summarize([row], scored=legacy)
|
|
assert result["recall"]["count"] == 1 and result["precision"]["rate"] == 1
|