marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 04:29:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(bench): archive reference-only scripts

Browse Source 
A1-A3 of the legacy inventory, moved with git mv to benchmarks/archive/:
the stage-2 verifier chain (verify_batched, score_batched, verify_sinks,
model_sinks, sink_candidates), the v2 tool hunter (hunt.py) and the
harnessx-removal equality proof. The plane's verify/agree/roles tasks
replaced the first six; the harnessx-removal spec is closed.

- The six independent scripts keep HERE = benchmarks/independent (population,
  evaluate.py, results) and add their own directory for archived siblings.
- harnessx_removal_equality's REPO root is one level deeper.
- tests/test_plane_scoring_parity.py loads score_batched from the archive
  (ran here, not skipped); test_harnessx_removal_equality points at the new path.
- benchmarks/archive/README.md: what each script was, which committed
  records cite it, the commit that last ran it. paths.md maps old -> new;
  committed records are not rewritten.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-02 08:07:35 +02:00
co-authored by Claude Opus 5.5
parent bd74ba63b2
commit 067adb9845
11 changed files with 81 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/independent/hunt.py
-191
View File
@@ -1,191 +0,0 @@
"""Run the UNCHANGED tool hunter over an independent population's pins, scoreable by the evaluation protocol.
The hunter (`tool_hunter.run_tool_hunter`, default prompt, default tools, default step budget) is handed the
regions the ranker selects for the case's family -- the same discovery and ranking the static scan starts from,
top `--max-regions` by rank, in groups of `--per-hunt` -- and nothing else: not the fix diff, not the advisory,
not the declared sites. Each pin's result is written in the scan-result shape `evaluate.py score` reads, into
`~/ousast-results/independent-<vN>-hunter/scans/`, so the same pre-registered matching scores it.
Model calls go to the resolved hunter client (DeepSeek when its key is set); the run stops, and says so, when
the recorded spend reaches `--budget-usd`.
Usage:
python benchmarks/independent/hunt.py --population population-v2.toml [--only ID,...] [--pins a,b]
python benchmarks/independent/hunt.py --population population-v2.toml --score
python benchmarks/independent/hunt.py --pilot-root <checkout> --family injection # a development case
"""
from __future__ import annotations
import argparse
import dataclasses
import json
import shutil
import sys
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import evaluate # noqa: E402
from openultrasast.mapping import analyze_entry_points # noqa: E402
from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402
from openultrasast.model.regions import regions_for # noqa: E402
from openultrasast.model.shipped import declared_sources # noqa: E402
from openultrasast.preprocess import build_file_target, enumerate_source_files # noqa: E402
# The model names a family in its own words; the protocol scores three. Fixed before any v2 pin is hunted.
FAMILY_WORDS = {
"untrusted_destination": ("ssrf", "server-side request", "open redirect", "redirect", "url fetch", "request forgery"),
"config_secrets": ("cors", "debug", "secret", "hardcoded", "hard-coded", "credential", "misconfig", "configuration", "tls", "certificate"),
"injection": ("inject", "sql", "sqli", "nosql", "command", "rce", "code execution", "eval", "template", "ssti", "xss", "deserializ", "ldap", "xpath"),
}
def family_of(raw: str, title: str) -> str:
text = f"{raw} {title}".lower()
for family, words in FAMILY_WORDS.items():
if any(word in text for word in words):
return family
return "other"
def hotspots_for(root: Path, family: str, max_regions: int) -> list:
from openultrasast.complexity.map import Hotspot
files = enumerate_source_files(root)
if not files or not sum(p.stat().st_size for p in files if p.is_file()):
raise SystemExit(f"read nothing under {root}")
targets = [build_file_target(root, p) for p in files]
regions = [
dataclasses.replace(r, families=tuple(f for f in r.families if f == family))
for r in regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root))
]
ranked = sorted((r for r in regions if r.families), key=lambda r: (-r.rank, r.path, r.function or ""))[:max_regions]
return [
Hotspot(
path=r.path,
function_name=r.function or None,
score=float(r.rank),
band="ranker",
signals={},
rationale=f"ranked region offering {family}",
test_hint=None,
inventory_finding_ids=(),
)
for r in ranked
]
def _spent(client: object) -> float:
cost = getattr(client, "cost_usd", None)
return float(cost()) if callable(cost) else 0.0
def hunt(root: Path, family: str, args: argparse.Namespace, spent: list[float]) -> dict:
from openultrasast import tool_hunter
client = tool_hunter.resolve_hunter_client()
if client is None:
raise SystemExit("no hunter client: set DEEPSEEK_API_KEY")
started = time.monotonic()
hotspots = hotspots_for(root, family, args.max_regions)
groups = [hotspots[i : i + args.per_hunt] for i in range(0, len(hotspots), args.per_hunt)]
findings, done, errors = [], 0, []
for group in groups:
if sum(spent) + _spent(client) >= args.budget_usd:
errors.append("budget_exhausted")
break
try:
found = tool_hunter.run_tool_hunter(root, group, client=client, model=args.model, max_steps=tool_hunter.DEFAULT_MAX_STEPS)
done += 1
except Exception as error: # noqa: BLE001 -- recorded per hunt; a pin with no completed hunt is an instrument failure
errors.append(f"{type(error).__name__}: {str(error)[:200]}")
continue
for f in found:
raw = next((t.split(":", 1)[1] for t in f.tags if t.startswith("family:")), "")
findings.append(
{
"site": f"{f.path}:{f.line or 0}:{f.function_name or ''}",
"family": family_of(raw, f.title),
"model_family": raw,
"title": f.title,
"witness": f.rationale[:400],
}
)
cost = _spent(client)
spent.append(cost)
return {
"root": str(root),
"families": [family],
"model": args.model,
"seconds": round(time.monotonic() - started, 1),
"questions": len(groups),
"completed": done,
"completed_regions": sorted({f"{h.path}:{h.function_name or ''}" for g in groups[:done] for h in g}),
"hunt_errors": errors,
"cost_usd": round(cost, 4),
"findings": sorted(findings, key=lambda f: f["site"]),
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--population", type=Path, default=evaluate.POPULATION)
parser.add_argument("--only", default="")
parser.add_argument("--pins", default="")
parser.add_argument("--pilot-root", type=Path)
parser.add_argument("--family", default="injection")
parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
parser.add_argument("--max-regions", type=int, default=20)
parser.add_argument("--per-hunt", type=int, default=5)
parser.add_argument("--budget-usd", type=float, default=10.0)
parser.add_argument("--score", action="store_true", help="score the hunter's results with the population's protocol")
args = parser.parse_args()
from openultrasast.config import load_config
load_config() # loads the project's .env: DeepSeek is the chat provider, OpenRouter only embeddings
spent: list[float] = []
if args.pilot_root:
result = hunt(args.pilot_root, args.family, args, spent)
print(json.dumps({k: v for k, v in result.items() if k != "findings"} | {"findings": result["findings"]}, indent=1))
return 0
evaluate.use(args.population)
evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-hunter")
if args.score:
return evaluate.score_v2()
only = {c.strip() for c in args.only.split(",") if c.strip()}
wanted = {p.strip() for p in args.pins.split(",") if p.strip()}
for case in evaluate.cases():
if only and case["id"] not in only:
continue
for label, commit in evaluate.pins(case).items():
if wanted and label not in wanted:
continue
out = evaluate.RESULTS / "scans" / f"{case['id']}--{label}.json"
if out.is_file():
continue
if sum(spent) >= args.budget_usd:
print(json.dumps({"stopped": "budget", "spent_usd": round(sum(spent), 4)}), flush=True)
return 1
out.parent.mkdir(parents=True, exist_ok=True)
checkout = evaluate.export(case, label, commit)
try:
result = hunt(checkout, case["family"], args, spent)
finally:
shutil.rmtree(checkout, ignore_errors=True)
if not result["completed"]:
result["instrument_error"] = "no hunt completed: " + "; ".join(result["hunt_errors"][:3])
result["root"] = "/case"
out.write_text(json.dumps(result, indent=2) + "\n")
print(
json.dumps({"case": case["id"], "pin": label, "findings": len(result["findings"]), "hunts": f"{result['completed']}/{result['questions']}", "usd": result["cost_usd"], "total_usd": round(sum(spent), 4)}),
flush=True,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/independent/model_sinks.py
-174
View File
@@ -1,174 +0,0 @@
"""Diagnostic: does a MODEL, reading source, find the operations the sink vocabulary misses?
Sink-first selection with the shipped vocabulary put the vulnerable function in scope in 2 of 17 v2 cases: the
real sinks (wp_redirect, download_url, spawn, ky, Session.get, parse_expr, Jinja render, Mongo find, JCR
createQuery, ...) are not in any token list, and each population brings new ones. This asks the model instead,
with no vocabulary: per product source file, which functions perform the family's dangerous operation on data
that may not be constant. Candidates are scored exactly as `sink_candidates.py` scores the vocabulary.
Source only. Files: the case's languages, product code (tests, docs, examples, vendored trees, minified bundles
and files over 200 kB excluded), sent in chunks of `--chunk-lines` numbered lines. A chunk whose call fails is
counted as UNCLASSIFIED, never as "no sinks". Hard spend ceiling `--budget-usd`; DeepSeek (`deepseek-flash`).
v2 is spent for tuning; this informs the selection design and qualifies nothing.
Usage: python benchmarks/independent/model_sinks.py [--population population-v2.toml] [--only ID,...]
"""
from __future__ import annotations
import argparse
import concurrent.futures
import json
import re
import shutil
import sys
import threading
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import evaluate # noqa: E402
from openultrasast.config import load_config # noqa: E402
from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402
from openultrasast.preprocess import detect_language, enumerate_source_files # noqa: E402
NOT_PRODUCT = re.compile(r"(^|/)(tests?|__tests__|spec|specs|docs?|examples?|fixtures?|e2e|cypress|storybook)/|\.(test|spec|stories)\.[jt]sx?$|_test\.py$|(^|/)test_[^/]*\.py$|\.min\.js$|(^|/)dist/|(^|/)build/")
OPERATIONS = {
"injection": (
"executes a database query (SQL, NoSQL, JCR/graph query), an operating-system command or shell, code or "
"expression evaluation (eval, exec, dynamic parsing into executable form), server-side template rendering "
"from a template string, or deserialization"
),
"untrusted_destination": "makes an outbound HTTP/network request, downloads a URL, or redirects the user to a URL",
"config_secrets": (
"sets security-relevant configuration: CORS policy, debug mode, TLS/certificate verification, cookie or "
"session security, or embeds a secret, password or API key"
),
}
PROMPT = (
"You are auditing source code. List every function in the code below that {operation}, where the query, "
"command, expression, template, URL or setting is not a compile-time constant. Answer with JSON only: "
'{{"functions": [{{"function": "<name, or <global> for top-level code>", "line": <line number>, '
'"operation": "<the call>"}}]}}. Return {{"functions": []}} if there is none.\n\nFile: {path}\n\n{code}'
)
def product_files(root: Path, languages: set[str]) -> list[Path]:
files = []
for path in enumerate_source_files(root):
relative = path.relative_to(root).as_posix()
if detect_language(path) in languages and not NOT_PRODUCT.search(relative) and path.stat().st_size <= 200_000:
files.append(path)
return files
def chunks(text: str, size: int) -> list[str]:
lines = text.splitlines()
return ["\n".join(f"{n + 1:5d}| {line}" for n, line in enumerate(lines[i : i + size], start=i)) for i in range(0, max(len(lines), 1), size)]
class Meter:
def __init__(self, ceiling: float) -> None:
self.ceiling, self.clients, self.lock = ceiling, [], threading.Lock()
def add(self, client: object) -> None:
with self.lock:
self.clients.append(client)
def spent(self) -> float:
with self.lock:
return sum(float(c.cost_usd()) for c in self.clients)
def classify(root: Path, path: Path, family: str, args: argparse.Namespace, meter: Meter, local: threading.local) -> tuple[list[tuple[str, str, int]], int, int]:
from openultrasast import tool_hunter
if not hasattr(local, "client"):
local.client = tool_hunter.resolve_hunter_client()
meter.add(local.client)
relative = path.relative_to(root).as_posix()
found, failed, asked = [], 0, 0
for code in chunks(path.read_text(errors="ignore"), args.chunk_lines):
if meter.spent() >= args.budget_usd:
return found, failed + 1, asked
asked += 1
prompt = PROMPT.format(operation=OPERATIONS[family], path=relative, code=code)
try:
response = local.client.complete(model=args.model, messages=[{"role": "user", "content": prompt}], tools=[], timeout_seconds=90, json_object=True)
items = json.loads(response.content or "{}").get("functions", [])
except Exception: # noqa: BLE001 -- counted as unclassified, never as "no sinks"
failed += 1
continue
for item in items if isinstance(items, list) else []:
if isinstance(item, dict) and isinstance(item.get("function"), str):
line = item.get("line") if isinstance(item.get("line"), int) else 0
found.append((relative, item["function"].strip() or "<global>", line))
return found, failed, asked
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml")
parser.add_argument("--only", default="")
parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
parser.add_argument("--chunk-lines", type=int, default=400)
parser.add_argument("--workers", type=int, default=8)
parser.add_argument("--budget-usd", type=float, default=8.0)
args = parser.parse_args()
load_config() # the project's .env: DeepSeek is the chat provider
evaluate.use(args.population)
evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-modelsinks")
evaluate.RESULTS.mkdir(parents=True, exist_ok=True)
only = {c.strip() for c in args.only.split(",") if c.strip()}
meter, local, rows = Meter(args.budget_usd), threading.local(), []
for case in evaluate.cases():
if only and case["id"] not in only:
continue
if meter.spent() >= args.budget_usd:
print(json.dumps({"stopped": "budget", "spent_usd": round(meter.spent(), 4)}), flush=True)
break
started, before = time.monotonic(), meter.spent()
checkout = evaluate.export(case, "vulnerable", case["vulnerable"])
try:
files = product_files(checkout, {case["language"], *case.get("also", [])})
found, failed, asked = [], 0, 0
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
for f, fail, ask in pool.map(lambda p: classify(checkout, p, case["family"], args, meter, local), files):
found += f
failed += fail
asked += ask
finally:
shutil.rmtree(checkout, ignore_errors=True)
functions = {(p, fn) for p, fn, _ in found}
changed = set(evaluate.hunks(case, "old"))
sites = {tuple(site.split("::", 1)) for site in case.get("sites", [])}
row = {
"id": case["id"], "family": case["family"], "files": len(files), "chunks": asked, "unclassified_chunks": failed,
"candidate_functions": len(functions), "candidate_files": len({p for p, _ in functions}),
"declared_site_in_candidates": sorted(f"{p}::{fn}" for p, fn in functions if (p, fn) in sites),
"fix_or_site_file_in_candidates": sorted({p for p, _ in functions if p in changed or p in {s[0] for s in sites}})[:5],
"usd": round(meter.spent() - before, 4), "seconds": round(time.monotonic() - started, 1),
} # fmt: skip
(evaluate.RESULTS / f"{case['id']}.json").write_text(json.dumps({**row, "candidates": sorted(found)}, indent=1) + "\n")
rows.append(row)
print(json.dumps(row), flush=True)
print(
json.dumps(
{
"cases": len(rows),
"declared_site_in_candidates": sum(bool(r["declared_site_in_candidates"]) for r in rows),
"fix_or_site_file_in_candidates": sum(bool(r["fix_or_site_file_in_candidates"]) for r in rows),
"spent_usd": round(meter.spent(), 4),
}
),
flush=True,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/independent/score_batched.py
-94
View File
@@ -1,94 +0,0 @@
"""Reference-only since 2026-09-30: the ai-service-plane tasks `plane/tasks/verify.py` and `agree.py`, run on google/ax
through `ousast plane run plane/runs/validation-46.yaml`, replace this script (benchmarks/independent/
plane-increment-2.json). It stays as the reference behaviour the plane was measured against.
Exploratory score of a batched-verifier run against the population's cases (protocol v2 matching).
The batched verifier writes one vulnerable-pin result per case (and a fixed-pin recheck with --fixed), not the
six pins protocol v2 scores, so this applies the same matching rule (`evaluate.matches_v2`) to what exists:
detected when an AGREED finding matches; unstable when only a DISPUTED one does; fixed-side alert when an agreed
finding at the fixed pin matches the fix's new side. The precision sample is every third agreed vulnerable-pin
finding (all if 15 or fewer), to be adjudicated by hand. v2 is spent for tuning: this qualifies nothing.
Usage: python benchmarks/independent/score_batched.py [--results-suffix batched] [--population population-v2.toml]
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import evaluate # noqa: E402
def anchored(result: dict) -> dict:
"""Sites anchored to the candidate asked about (older records used the model's reported location)."""
for key in ("findings", "disputed"):
for f in result.get(key, []):
path, _, function = f["candidate"].partition("::")
site_path, _, rest = f["site"].partition(":")
line = rest.split(":")[0] if site_path == path else "0"
f["site"] = f"{path}:{line}:{function}"
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml")
parser.add_argument("--results-suffix", default="batched")
args = parser.parse_args()
evaluate.use(args.population)
results = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-" + args.results_suffix)
scans = results / "scans"
per_case, pool = [], []
for case in evaluate.cases():
path = scans / f"{case['id']}--vulnerable_a.json"
if not path.is_file():
per_case.append({"id": case["id"], "status": "incomplete"})
continue
vulnerable = anchored(json.loads(path.read_text()))
old, new = evaluate.hunks(case, "old"), evaluate.hunks(case, "new")
agreed = [f["site"] for f in vulnerable["findings"] if evaluate.matches_v2(f, case, old)]
disputed = [f["site"] for f in vulnerable.get("disputed", []) if evaluate.matches_v2(f, case, old)]
fixed_path = scans / f"{case['id']}--fixed_a.json"
fixed = anchored(json.loads(fixed_path.read_text())) if fixed_path.is_file() else None
fixed_alerts = [f["site"] for f in fixed["findings"] if evaluate.matches_v2(f, case, new)] if fixed else None
per_case.append(
{
"id": case["id"], "family": case["family"], "detected": bool(agreed), "detected_sites": agreed,
"unstable": bool(disputed) and not agreed, "unstable_sites": disputed,
"agreed_findings": len(vulnerable["findings"]), "disputed_findings": len(vulnerable.get("disputed", [])),
"triaged_out": vulnerable.get("triaged_out"), "candidates": vulnerable.get("questions"),
"fixed_side_alerts": fixed_alerts, "fixed_agreed_total": len(fixed["findings"]) if fixed else None,
"removed_by_fix": fixed.get("removed_by_fix") if fixed else None, "usd": vulnerable.get("usd"), "usage": vulnerable.get("usage"),
}
) # fmt: skip
pool += [{"case": case["id"], **f} for f in vulnerable["findings"]]
pool.sort(key=lambda f: (f["case"], f["site"]))
sample = pool if len(pool) <= 15 else pool[0::3]
scored = [c for c in per_case if c.get("status") != "incomplete"]
detected = sum(c["detected"] for c in scored)
with_fixed = [c for c in scored if c["fixed_side_alerts"] is not None]
result = {
"protocol": "protocol-v2.md matching, exploratory (batched verifier, vulnerable + fixed pins only)",
"cases_scored": len(scored), "cases_incomplete": [c["id"] for c in per_case if c.get("status") == "incomplete"],
"recall": [detected, len(scored)], "recall_wilson95": evaluate.wilson(detected, len(scored)),
"unstable_detections": [c["id"] for c in scored if c["unstable"]],
"fixed_side_alerts": sum(len(c["fixed_side_alerts"]) for c in with_fixed) if with_fixed else None,
"fixed_pins_rechecked": len(with_fixed),
"pooled_agreed_findings": len(pool), "precision_sample_size": len(sample),
"usd": round(sum(float(c.get("usd") or 0) for c in scored), 2),
"cases": per_case, "precision_sample": sample,
} # fmt: skip
(results / "score.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps({k: v for k, v in result.items() if k not in ("cases", "precision_sample")}, indent=1))
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/independent/sink_candidates.py
-121
View File
@@ -1,121 +0,0 @@
"""Diagnostic: would SINK-FIRST selection, from source alone, put the vulnerable function in scope?
The v2 hunter experiment showed scope selection is the bottleneck: the ranker's top 20 entry-point regions held
the vulnerable file in 1 of 17 cases. This asks the opposite direction without any engine or model: at each
case's vulnerable pin, every call matching the family's shipped sink vocabulary (`taint_specs`) in product
source is a candidate, attributed to its enclosing function by a per-language declaration scan. It reports how
many candidates a repository yields (the budget a hunter would need) and whether a declared site function, or
at least a file the fix changes, is among them.
Source only: the checkout is the pin's tree; git is used to export it and, for the score, to read the fix's
changed files -- never to choose scope.
v2 is spent for tuning; this informs the selection design and qualifies nothing.
Usage: python benchmarks/independent/sink_candidates.py [--population population-v2.toml] [--only ID,...]
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import evaluate # noqa: E402
from openultrasast.model.specs import taint_specs # noqa: E402
from openultrasast.preprocess import detect_language, enumerate_source_files # noqa: E402
DECLARATION = {
"python": re.compile(r"^\s*(?:async\s+)?def\s+(\w+)"),
"php": re.compile(r"^\s*(?:(?:public|private|protected|static|final|abstract)\s+)*function\s+&?(\w+)"),
"javascript": re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s*\*?\s*(\w+)|^\s*(?:(?:public|private|protected|static|async)\s+)*(\w+)\s*\([^;]*\)\s*(?::[^{=]+)?\{\s*$|^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:function|\()"),
}
DECLARATION["typescript"] = DECLARATION["javascript"]
KEYWORDS = {"if", "for", "while", "switch", "catch", "return", "function", "else"}
def token(name: str) -> re.Pattern[str]:
before = r"(?<![A-Za-z0-9_$])" if name[:1].isalnum() or name[:1] in "_$" else ""
return re.compile(before + re.escape(name) + r"\s*\(")
def enclosing(lines: list[str], index: int, language: str) -> str:
pattern = DECLARATION.get(language)
if pattern is None:
return ""
for back in range(index, -1, -1):
match = pattern.match(lines[back])
if match:
name = next((g for g in match.groups() if g), "")
if name and name not in KEYWORDS:
return name
return "<global>"
def candidates(root: Path, family: str) -> dict[tuple[str, str], list[int]]:
found: dict[tuple[str, str], list[int]] = {}
patterns: dict[str, list[re.Pattern[str]]] = {}
for path in enumerate_source_files(root):
language = detect_language(path)
if language not in patterns:
spec = taint_specs(language=language).get(family) if language != "unknown" else None
patterns[language] = [token(n) for n in spec.sinks] if spec else []
if not patterns[language]:
continue
try:
lines = path.read_text(errors="ignore").splitlines()
except OSError:
continue
relative = path.relative_to(root).as_posix()
for index, line in enumerate(lines):
stripped = line.strip()
if stripped.startswith(("//", "#", "*", "/*")):
continue
if any(p.search(line) for p in patterns[language]):
found.setdefault((relative, enclosing(lines, index, language)), []).append(index + 1)
return found
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml")
parser.add_argument("--only", default="")
args = parser.parse_args()
evaluate.use(args.population)
evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-sinkfirst")
only = {c.strip() for c in args.only.split(",") if c.strip()}
rows = []
for case in evaluate.cases():
if only and case["id"] not in only:
continue
checkout = evaluate.export(case, "vulnerable", case["vulnerable"])
try:
found = candidates(checkout, case["family"])
finally:
shutil.rmtree(checkout, ignore_errors=True)
changed = set(evaluate.hunks(case, "old"))
sites = {tuple(site.split("::", 1)) for site in case.get("sites", [])}
site_hit = sorted(f"{p}::{f}" for p, f in found if (p, f) in sites)
file_hit = sorted({p for p, _ in found if p in changed or p in {s[0] for s in sites}})
row = {
"id": case["id"], "family": case["family"], "candidate_functions": len(found),
"candidate_files": len({p for p, _ in found}), "declared_site_in_candidates": site_hit,
"fix_or_site_file_in_candidates": file_hit[:5],
} # fmt: skip
rows.append(row)
print(json.dumps(row), flush=True)
sites_in = sum(bool(r["declared_site_in_candidates"]) for r in rows)
files_in = sum(bool(r["fix_or_site_file_in_candidates"]) for r in rows)
print(json.dumps({"cases": len(rows), "declared_site_in_candidates": sites_in, "fix_or_site_file_in_candidates": files_in}))
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/independent/verify_batched.py
-289
View File
@@ -1,289 +0,0 @@
"""Reference-only since 2026-09-30: the ai-service-plane tasks `plane/tasks/verify.py` and `agree.py`, run on google/ax
through `ousast plane run plane/runs/validation-46.yaml`, replace this script (benchmarks/independent/
plane-increment-2.json). It stays as the reference behaviour the plane was measured against.
Stage 2, second design: triage per file, verify per file, report only what two passes agree on.
The first verifier (`verify_sinks.py`) asked one 6-step tool hunt per candidate function. Measured on pgAdmin
(verifier-check-2026-09-29.json): $0.014-0.020 per candidate, and on 180 candidates verified twice the two runs
flagged 15 and 13 with only 2 in common -- single-pass verdicts on borderline candidates flip. This design
changes three things, each checked against the 15 declared sites and those 26 flickering candidates:
1. TRIAGE, one no-tools call per file: the file's relevant chunks and its candidate functions, asking where the
operation's data ORIGINATES. `request`, `parameter` and `stored` go on to verification; `internal` and
`constant` do not; anything unclassified goes on (a failed call is never a "safe" verdict).
2. BATCHED verification, one 6-step tool hunt per file for up to `--per-hunt` surviving candidates, with one
verdict per listed operation. The file read and the caller searches are shared instead of repeated ~5 times.
3. AGREEMENT: `--passes` independent hunts per file (fresh conversations); a candidate is reported only when
every pass flags it. Findings from single passes are kept in the record as `disputed`.
Source only; the stage-1 candidates come from `model_sinks.py`. Results in `~/ousast-results/independent-<vN>-
<suffix>/scans/` in the shape `evaluate.py` scores; a per-case `.jsonl` makes a stopped run resumable per file.
EXPLORATORY: v2 is spent for tuning; this qualifies nothing.
Usage:
python benchmarks/independent/verify_batched.py [--only ID,...] [--subset sets.json] [--passes 2] [--budget-usd 40]
"""
from __future__ import annotations
import argparse
import concurrent.futures
import json
import shutil
import sys
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import evaluate # noqa: E402
from model_sinks import OPERATIONS, chunks # noqa: E402
from verify_sinks import ACCOUNT_ERRORS, _Recording # noqa: E402
from openultrasast.config import load_config # noqa: E402
from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402
VERIFY = {"request", "parameter", "stored"}
TRIAGE = (
"Below is a source file with numbered lines, and a list of functions in it that each {operation}. For EACH "
"listed function, say where the data used by that operation ORIGINATES, tracing within this file only:\n"
"- request: an HTTP request (parameters, body, headers, cookies, uploads), a webhook or API payload;\n"
"- parameter: a parameter of the function whose callers are not in this file;\n"
"- stored: a database row, file or setting written earlier, possibly from external input;\n"
"- internal: values the program computes from constants, the operator's environment, CLI arguments or "
"trusted framework state;\n"
"- constant: a literal.\n"
'Answer with JSON only: {{"functions": [{{"function": "<name exactly as listed>", "data_origin": "<one of the five>"}}]}}.'
"\n\nFunctions: {functions}\n\nFile: {path}\n\n{code}"
)
SYSTEM = (
"You are a security reviewer with repository tools (read_file, grep_repo, find_refs). You are given several "
"operations in ONE file, each named by its function. For EACH operation decide whether data an attacker "
"controls -- HTTP request parameters, body, headers, cookies, uploaded or imported content, webhook or API "
"payloads, values stored earlier from such input -- can reach it without a sufficient guard (validation or "
"allowlisting, escaping that fits the exact context, parameterization, or a fixed destination). Use the tools "
"to follow callers and data. Report an operation only if you can state the concrete path from the source to "
"it. Answer with a JSON array: one object per vulnerable operation with path, line, function_name (exactly as "
"listed), title and rationale (the source, each hop, and why no guard applies); [] if none is vulnerable or "
"you cannot establish a path."
)
_CLIENT = None
def _client():
global _CLIENT
if _CLIENT is None:
from openultrasast import tool_hunter
load_config()
_CLIENT = tool_hunter.resolve_hunter_client()
return _CLIENT
def _relevant_code(text: str, lines: list[int], chunk_lines: int, max_chunks: int = 6) -> str:
parts = chunks(text, chunk_lines)
if not lines or any(line <= 0 for line in lines) or len(parts) <= max_chunks:
return "\n".join(parts[:max_chunks])
wanted = sorted({min((line - 1) // chunk_lines, len(parts) - 1) for line in lines})
return "\n".join(parts[i] for i in wanted[:max_chunks])
def triage(root: Path, path: str, candidates: list[list], family: str, model: str, chunk_lines: int) -> dict[str, str]:
"""Function name -> data_origin for this file's candidates; a name missing from the answer is unclassified."""
client = _client()
try:
text = (root / path).read_text(errors="ignore")
except OSError:
return {}
names = sorted({fn for _, fn, _ in candidates})
prompt = TRIAGE.format(operation=OPERATIONS[family], functions=json.dumps(names), path=path, code=_relevant_code(text, [ln for _, _, ln in candidates], chunk_lines))
response = client.complete(model=model, messages=[{"role": "user", "content": prompt}], tools=[], timeout_seconds=90, json_object=True)
try:
items = json.loads(response.content or "{}").get("functions", [])
except (json.JSONDecodeError, AttributeError):
return {}
return {str(i.get("function")): str(i.get("data_origin", "")).strip().lower() for i in items if isinstance(i, dict) and i.get("function") in names}
def hunt(root: Path, path: str, group: list[list], family: str, model: str, steps: int) -> tuple[list[dict], dict]:
from openultrasast import tool_hunter
from openultrasast.complexity.map import Hotspot
recording = _Recording(_client())
listing = "\n".join(f"- function `{fn}` around line {ln}" for _, fn, ln in group)
prompt = (
f"Operations to judge, all in `{path}`, each of which {OPERATIONS[family]}:\n{listing}\n"
"The file is above. For each, trace where the operation's data comes from and whether it is guarded."
)
spots = [Hotspot(path=path, function_name=fn, score=1.0, band="candidate", signals={}, rationale="model-classified sink", test_hint=None, inventory_finding_ids=()) for _, fn, _ in group]
found = tool_hunter.run_tool_hunter(root, spots, client=recording, model=model, max_steps=steps, system_prompt=SYSTEM, user_prompt=prompt, context_files=[path], tags=("sink-verifier",))
names = {fn for _, fn, _ in group}
rows = []
for f in found:
fn = f.function_name or ""
if fn not in names: # attribute by the nearest listed line when the model renamed the function
fn = min(group, key=lambda c: abs((c[2] or 0) - (f.line or 0)))[1]
# The verdict is about the candidate operation, so the site is the candidate's file and function; the
# model's own location is kept beside it (it often names the file where the input enters instead).
line = f.line if f.path == path else next(c[2] for c in group if c[1] == fn)
rows.append({"candidate": f"{path}::{fn}", "site": f"{path}:{line or 0}:{fn}", "reported_at": f"{f.path}:{f.line or 0}", "family": family, "title": f.title, "witness": f.rationale[:600]})
return rows, recording.usage()
def verify_file(root: str, path: str, candidates: list[list], family: str, args_: dict) -> dict:
"""One file, in a worker process: triage, then `passes` batched hunts over the survivors."""
client = _client()
before = float(client.cost_usd())
record: dict = {"path": path, "candidates": candidates, "triage": {}, "passes": [], "usage": {}}
try:
if not args_["no_triage"]:
record["triage"] = triage(Path(root), path, candidates, family, args_["model"], args_["chunk_lines"])
kept = [c for c in candidates if record["triage"].get(c[1], "unclassified") in VERIFY | {"unclassified"} or record["triage"].get(c[1]) not in {"internal", "constant"}]
record["kept"] = [c[1] for c in kept]
usage: dict[str, int] = {}
for _pass in range(args_["passes"]):
flagged: list[dict] = []
for i in range(0, len(kept), args_["per_hunt"]):
rows, used = hunt(Path(root), path, kept[i : i + args_["per_hunt"]], family, args_["model"], args_["max_steps"])
flagged += rows
usage = {k: usage.get(k, 0) + used.get(k, 0) for k in set(usage) | set(used)}
record["passes"].append(flagged)
record["usage"] = usage
except Exception as error: # noqa: BLE001 -- recorded; an unverified file is not a clean one
record["unverified"] = f"{type(error).__name__}: {str(error)[:160]}"
record["usd"] = round(float(client.cost_usd()) - before, 4)
return record
def agreement(record: dict) -> tuple[list[dict], list[dict]]:
"""(agreed, disputed): flagged in every pass vs in some."""
if not record.get("passes"):
return [], []
by_pass = [{r["candidate"]: r for r in flagged} for flagged in record["passes"]]
every = set.intersection(*(set(b) for b in by_pass))
some = set.union(*(set(b) for b in by_pass))
first = by_pass[0]
pick = lambda c: next(b[c] for b in by_pass if c in b) # noqa: E731
return [dict(first[c], passes=len(by_pass)) for c in sorted(every)], [dict(pick(c), passes=sum(c in b for b in by_pass)) for c in sorted(some - every)]
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml")
parser.add_argument("--only", default="")
parser.add_argument("--subset", type=Path, help='JSON {case_id: [[path, function], ...]}: verify only these candidates')
parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
parser.add_argument("--max-steps", type=int, default=6)
parser.add_argument("--per-hunt", type=int, default=6)
parser.add_argument("--passes", type=int, default=2)
parser.add_argument("--chunk-lines", type=int, default=400)
parser.add_argument("--no-triage", action="store_true")
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--budget-usd", type=float, default=40.0)
parser.add_argument("--results-suffix", default="batched")
parser.add_argument("--fixed", action="store_true", help="recheck the AGREED candidates of each case at its fixed pin (no triage)")
args = parser.parse_args()
load_config()
evaluate.use(args.population)
candidates_dir = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-modelsinks")
evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-" + args.results_suffix)
out_dir = evaluate.RESULTS / "scans"
out_dir.mkdir(parents=True, exist_ok=True)
only = {c.strip() for c in args.only.split(",") if c.strip()}
subset = json.loads(args.subset.read_text()) if args.subset else None
settings = {k: getattr(args, k) for k in ("model", "max_steps", "per_hunt", "passes", "chunk_lines", "no_triage")}
spent = 0.0
for case in evaluate.cases():
if only and case["id"] not in only or (subset is not None and case["id"] not in subset):
continue
label = "fixed_a" if args.fixed else "vulnerable_a"
out = out_dir / f"{case['id']}--{label}.json"
if out.is_file():
continue
log = out_dir / f"{case['id']}--{label}.jsonl"
done = {r["path"]: r for r in (json.loads(row) for row in log.read_text().splitlines()) if "unverified" not in r} if log.is_file() else {}
spent += sum(float(r.get("usd", 0.0)) for r in done.values())
record = json.loads((candidates_dir / f"{case['id']}.json").read_text())
unique = list({(p, fn): [p, fn, ln] for p, fn, ln in sorted(record["candidates"])}.values())
if subset is not None:
wanted = {tuple(x) for x in subset[case["id"]]}
unique = [c for c in unique if (c[0], c[1]) in wanted]
if args.fixed:
# The agreed findings at the vulnerable pin, by (path, function), asked again at the fixed pin. Line
# numbers are the vulnerable pin's; the file itself is in the prompt. A function the fix removed
# cannot be asked and counts as removed.
vulnerable_out = out_dir / f"{case['id']}--vulnerable_a.json"
if not vulnerable_out.is_file():
continue
agreed = {tuple(f["candidate"].split("::", 1)) for f in json.loads(vulnerable_out.read_text())["findings"]}
unique = [c for c in unique if (c[0], c[1]) in agreed]
settings = dict(settings, no_triage=True)
by_file: dict[str, list[list]] = {}
for c in unique:
by_file.setdefault(c[0], []).append(c)
pending = {p: cs for p, cs in by_file.items() if p not in done}
started = time.monotonic()
checkout = evaluate.export(case, label, case["fixed" if args.fixed else "vulnerable"])
if args.fixed:
present = [c for c in unique if (checkout / c[0]).is_file()]
removed = [f"{c[0]}::{c[1]}" for c in unique if (checkout / c[0]).is_file() is False]
unique, by_file = present, {}
for c in unique:
by_file.setdefault(c[0], []).append(c)
pending = {p: cs for p, cs in by_file.items() if p not in done}
account_error = ""
try:
with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as pool, log.open("a") as sink:
futures = {pool.submit(verify_file, str(checkout), p, cs, case["family"], settings): p for p, cs in pending.items()}
for future in concurrent.futures.as_completed(futures):
if future.cancelled():
continue
rec = future.result()
if any(code in str(rec.get("unverified", "")) for code in ACCOUNT_ERRORS):
account_error = str(rec["unverified"])[:200]
for other in futures:
other.cancel()
continue
spent += float(rec.get("usd", 0.0))
sink.write(json.dumps(rec) + "\n")
sink.flush()
if "unverified" not in rec:
done[rec["path"]] = rec
if spent >= args.budget_usd:
for other in futures:
other.cancel()
finally:
shutil.rmtree(checkout, ignore_errors=True)
agreed, disputed = [], []
for rec in done.values():
a, d = agreement(rec)
agreed += a
disputed += d
triaged_out = sum(len(r["candidates"]) - len(r.get("kept", [])) for r in done.values())
usage = {k: sum(int(r.get("usage", {}).get(k, 0)) for r in done.values()) for k in ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens", "calls")}
complete = len(done) >= len(by_file) and not account_error
result = {
"root": "/case", "families": [case["family"]], "questions": len(unique), "completed": sum(len(r["candidates"]) for r in done.values()),
"completed_regions": sorted(f"{c[0]}:{c[1]}" for r in done.values() for c in r["candidates"]),
"triaged_out": triaged_out, "passes": args.passes, "usage": usage, "usd": round(sum(float(r.get("usd", 0.0)) for r in done.values()), 4),
"seconds": round(time.monotonic() - started, 1), "findings": sorted(agreed, key=lambda f: f["site"]), "disputed": sorted(disputed, key=lambda f: f["site"]),
} # fmt: skip
if args.fixed:
result["removed_by_fix"] = removed
if complete:
out.write_text(json.dumps(result, indent=1) + "\n")
print(json.dumps({"case": case["id"], "files": f"{len(done)}/{len(by_file)}", "candidates": len(unique), "triaged_out": triaged_out, "agreed": len(agreed), "disputed": len(disputed), "usd": result["usd"], "total_usd": round(spent, 4), "complete": complete}), flush=True)
if account_error:
print(json.dumps({"stopped": "account", "error": account_error}), flush=True)
return 2
if spent >= args.budget_usd:
print(json.dumps({"stopped": "budget", "spent_usd": round(spent, 4)}), flush=True)
break
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/independent/verify_sinks.py
-233
View File
@@ -1,233 +0,0 @@
"""Stage 2 of source-only, model-driven detection: verify each model-classified sink candidate.
`model_sinks.py` put the vulnerable function among the candidates in 15 of 17 v2 cases, among ~4,800 candidates.
This asks, per candidate, one bounded question with repository tools (the existing tool hunter with its
per-family overrides): can attacker-controlled data reach THIS operation without a sufficient guard? A candidate
is reported only when the model states a concrete source-to-sink path and names no sufficient guard.
Scope: the candidates `model_sinks.py` recorded for each case's vulnerable pin
(~/ousast-results/independent-v2-modelsinks/<case>.json). One run per pin; the fixed pin is rechecked only for
detected sites (`--fixed`). Findings are written in the scan-result shape `evaluate.py` reads. Hard spend
ceiling. EXPLORATORY: v2 is spent for tuning; this qualifies nothing.
Usage: python benchmarks/independent/verify_sinks.py [--only ID,...] [--budget-usd 40] [--fixed]
"""
from __future__ import annotations
import argparse
import concurrent.futures
import json
import shutil
import sys
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import evaluate # noqa: E402
from model_sinks import OPERATIONS # noqa: E402
from openultrasast.config import load_config # noqa: E402
from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402
# The provider refusing the ACCOUNT (no credit, bad key): every later call fails the same way.
ACCOUNT_ERRORS = ("HTTP Error 402", "HTTP Error 401", "Insufficient Balance")
SYSTEM = (
"You are a security reviewer with repository tools (read_file, grep_repo, find_refs). You are given ONE "
"operation in ONE function. Decide whether data an attacker controls -- HTTP request parameters, body, "
"headers, cookies, uploaded or imported content, webhook or API payloads, values stored earlier from such "
"input -- can reach that operation without a sufficient guard (validation or allowlisting, escaping that fits "
"the exact context, parameterization, or a fixed destination). Use the tools to follow callers and data. "
"Report it only if you can state the concrete path from the source to the operation. Answer with a JSON array: "
"one object with path, line, function_name, title, rationale (the source, each hop, and why no guard applies) "
"if it is vulnerable; [] if it is not, or if you cannot establish the path."
)
_CLIENT = None
def _client():
global _CLIENT
if _CLIENT is None:
from openultrasast import tool_hunter
load_config()
_CLIENT = tool_hunter.resolve_hunter_client()
return _CLIENT
class _Recording:
"""Wraps the chat client for one candidate: counts investigation turns and slices token usage."""
def __init__(self, client) -> None:
self.client, self.turns_with_tools, self.first_usage = client, 0, len(getattr(client, "usage", []))
def complete(self, **kw):
response = self.client.complete(**kw)
if response.tool_calls:
self.turns_with_tools += 1
return response
def usage(self) -> dict[str, int]:
rows = getattr(self.client, "usage", [])[self.first_usage :]
keys = ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens")
return {k: sum(int(r.get(k) or 0) for r in rows) for k in keys} | {"calls": len(rows)}
def _hunt(root: str, spot, prompt: str, path: str, model: str, steps: int):
from openultrasast import tool_hunter
recording = _Recording(_client())
found = tool_hunter.run_tool_hunter(
Path(root), [spot], client=recording, model=model, max_steps=steps,
system_prompt=SYSTEM, user_prompt=prompt, context_files=[path], tags=("sink-verifier",),
) # fmt: skip
return found, recording
def verify(root: str, candidate: list, family: str, model: str, max_steps: int, escalate_steps: int = 0) -> dict:
"""One candidate, in a worker process: CPU-bound repository tools do not serialise behind one interpreter.
Adaptive budget: the hunt runs at `max_steps`; if it spent EVERY step on tool calls and reported nothing, the
model was still investigating when the budget ended, and the hunt is repeated once at `escalate_steps`. A hunt
that concluded early (fewer tool turns than steps) is final at the small budget. Measured on the seven declared
v2 sites: 3 steps found 5, 8 steps found 6; most candidates conclude in 3.
"""
from openultrasast.complexity.map import Hotspot
client = _client()
before = float(client.cost_usd())
path, function, line = candidate
spot = Hotspot(path=path, function_name=function, score=1.0, band="candidate", signals={}, rationale="model-classified sink", test_hint=None, inventory_finding_ids=())
prompt = (
f"Operation to judge: in `{path}`, function `{function}`, around line {line}, code that {OPERATIONS[family]}. "
"The file is above. Trace where the operation's data comes from and whether it is guarded."
)
usage: dict[str, int] = {}
escalated = False
try:
found, recording = _hunt(root, spot, prompt, path, model, max_steps)
usage = recording.usage()
if not found and escalate_steps > max_steps and recording.turns_with_tools >= max_steps:
escalated = True
found, recording = _hunt(root, spot, prompt, path, model, escalate_steps)
second = recording.usage()
usage = {k: usage.get(k, 0) + second.get(k, 0) for k in set(usage) | set(second)}
except Exception as error: # noqa: BLE001 -- recorded; an unverified candidate is not a clean one
return {"candidate": candidate, "unverified": f"{type(error).__name__}: {str(error)[:120]}", "usd": float(client.cost_usd()) - before}
findings = [
{"site": f"{f.path}:{f.line or line}:{f.function_name or function}", "family": family, "title": f.title, "witness": f.rationale[:600], "candidate": f"{path}::{function}"}
for f in found
]
return {"candidate": candidate, "findings": findings, "usd": float(client.cost_usd()) - before, "escalated": escalated, "usage": usage}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml")
parser.add_argument("--only", default="")
parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
parser.add_argument("--max-steps", type=int, default=6)
parser.add_argument("--escalate-steps", type=int, default=0, help="rerun a hunt that used every step without a verdict; 0 disables (measured: 88% escalate, +45% cost)")
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--budget-usd", type=float, default=40.0)
parser.add_argument("--fixed", action="store_true", help="recheck detected sites at the fixed pin")
parser.add_argument("--results-suffix", default="verified", help="results go to independent-<vN>-<suffix>/")
parser.add_argument("--declared-sites-only", action="store_true", help="verify only the case's declared sites (a known-answer check)")
parser.add_argument("--candidates-from", type=Path, help="verify exactly the candidates recorded in this per-case .jsonl")
args = parser.parse_args()
load_config()
evaluate.use(args.population)
candidates_dir = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-modelsinks")
# Checkouts too, not only results: two runs exporting the same pin into one path deleted each other's tree.
evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-" + args.results_suffix)
out_dir = evaluate.RESULTS / "scans"
out_dir.mkdir(parents=True, exist_ok=True)
only = {c.strip() for c in args.only.split(",") if c.strip()}
spent = 0.0
for case in evaluate.cases():
if only and case["id"] not in only:
continue
label = "fixed_a" if args.fixed else "vulnerable_a"
out = out_dir / f"{case['id']}--{label}.json"
if out.is_file():
continue
log = out_dir / f"{case['id']}--{label}.jsonl"
done = [json.loads(row) for row in log.read_text().splitlines()] if log.is_file() else []
spent += sum(float(d.get("usd", 0.0)) for d in done)
record = json.loads((candidates_dir / f"{case['id']}.json").read_text())
todo = list({(p, fn): (p, fn, ln) for p, fn, ln in sorted(record["candidates"])}.values()) # one question per function
if args.declared_sites_only:
sites = {tuple(site.split("::", 1)) for site in case.get("sites", [])}
todo = [c for c in todo if (c[0], c[1]) in sites]
if args.candidates_from:
wanted = {tuple(json.loads(row)["candidate"][:2]) for row in args.candidates_from.read_text().splitlines() if row.strip()}
todo = [c for c in todo if (c[0], c[1]) in wanted]
if args.fixed:
vulnerable = json.loads((out_dir / f"{case['id']}--vulnerable_a.json").read_text())
detected = {f["candidate"] for f in vulnerable["findings"]}
todo = [c for c in todo if f"{c[0]}::{c[1]}" in detected]
# Only a VERIFIED candidate is done. An unverified one -- a failed call -- is asked again on resume; counting
# it as done wrote sixteen cases as complete with zero calls made when the account ran out of credit.
done = [d for d in done if "findings" in d]
finished = {tuple(d["candidate"][:2]) for d in done}
pending = [c for c in todo if (c[0], c[1]) not in finished]
started = time.monotonic()
checkout = evaluate.export(case, label, case["fixed" if args.fixed else "vulnerable"])
stopped, account_error = False, ""
try:
with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as pool, log.open("a") as sink:
futures = []
for candidate in pending:
futures.append(pool.submit(verify, str(checkout), list(candidate), case["family"], args.model, args.max_steps, args.escalate_steps))
for future in concurrent.futures.as_completed(futures):
if future.cancelled(): # cancelled at the ceiling: never run, nothing to record
continue
result = future.result()
if any(code in str(result.get("unverified", "")) for code in ACCOUNT_ERRORS):
# The account, not the candidate: nothing after this can succeed. Stop, record nothing.
account_error = str(result["unverified"])[:200]
stopped = True
for other in futures:
other.cancel()
continue
spent += float(result.get("usd", 0.0))
sink.write(json.dumps(result) + "\n")
sink.flush()
done.append(result)
if spent >= args.budget_usd and not stopped:
stopped = True
for other in futures:
other.cancel()
finally:
shutil.rmtree(checkout, ignore_errors=True)
verified = {tuple(d["candidate"][:2]) for d in done if "findings" in d}
findings = [f for d in done for f in d.get("findings", [])]
unverified = [d["unverified"] for d in done if "unverified" in d]
complete = len(verified) >= len(todo) and not account_error
result = {
"root": "/case", "families": [case["family"]], "questions": len(todo), "completed": len(verified),
"completed_regions": sorted(f"{p}:{fn}" for p, fn in verified), "unverified": unverified[:20],
"usd": round(sum(float(d.get("usd", 0.0)) for d in done), 4), "seconds": round(time.monotonic() - started, 1),
"findings": sorted(findings, key=lambda f: f["site"]),
} # fmt: skip
if complete:
out.write_text(json.dumps(result, indent=1) + "\n")
tokens = {k: sum(int(d.get("usage", {}).get(k, 0)) for d in done) for k in ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens")}
result["usage"] = tokens | {"escalated": sum(1 for d in done if d.get("escalated"))}
print(json.dumps({"case": case["id"], "pin": label, "candidates": len(todo), "verified": len(verified), "findings": len(findings), "unverified": len(unverified), "escalated": result["usage"]["escalated"], "cache_share": round(tokens["prompt_cache_hit_tokens"] / max(tokens["prompt_tokens"], 1), 2), "usd": result["usd"], "total_usd": round(spent, 4), "complete": complete}), flush=True)
if account_error:
print(json.dumps({"stopped": "account", "error": account_error, "spent_usd": round(spent, 4)}), flush=True)
return 2
if stopped or spent >= args.budget_usd:
print(json.dumps({"stopped": "budget", "spent_usd": round(spent, 4)}), flush=True)
break
return 0
if __name__ == "__main__":
raise SystemExit(main())