From 6bfb05f952edbd40111cdf9a074fca58420c0fa1 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:10:02 +0200 Subject: [PATCH] spec(pre-push-safety-net): pre-register the hunter experiment on population v2 This is the deciding experiment of the strategy review, fixed before any v2 pin is hunted. - Hunter: the unchanged tool hunter (default prompt, tools and 4 steps) on deepseek-flash. - Scope: the ranker's top 20 regions for the case family, in four hunts. The model never sees the fix diff or the declared sites. - Families: a fixed keyword table maps the model's family to the protocol's three. - Scoring: exactly protocol v2's scorer. - Cost: a hard $10 ceiling. hunt.py runs it and scores it (--score). A pilot on a v1 development case (mongo-express) found the vulnerability at lib/bson.js:60, at $0.031 per pin, alongside a lot of speculative noise. Transport fix found by the pilot: the DeepSeek endpoint the hunter builds was unpriced, so the cost meter read $0.00 for any number of calls. It is now priced. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- benchmarks/independent/hunt.py | 191 +++++++++++++++++++ benchmarks/independent/hunter-protocol-v2.md | 55 ++++++ src/openultrasast/tool_hunter.py | 5 +- tests/test_model_endpoint.py | 2 + 4 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 benchmarks/independent/hunt.py create mode 100644 benchmarks/independent/hunter-protocol-v2.md diff --git a/benchmarks/independent/hunt.py b/benchmarks/independent/hunt.py new file mode 100644 index 0000000..08837da --- /dev/null +++ b/benchmarks/independent/hunt.py @@ -0,0 +1,191 @@ +"""Run the UNCHANGED tool hunter over an independent population's pins, scoreable by the evaluation protocol. + +The hunter (`tool_hunter.run_tool_hunter`, default prompt, default tools, default step budget) is handed the +regions the ranker selects for the case's family -- the same discovery and ranking the static scan starts from, +top `--max-regions` by rank, in groups of `--per-hunt` -- and nothing else: not the fix diff, not the advisory, +not the declared sites. Each pin's result is written in the scan-result shape `evaluate.py score` reads, into +`~/ousast-results/independent--hunter/scans/`, so the same pre-registered matching scores it. + +Model calls go to the resolved hunter client (DeepSeek when its key is set); the run stops, and says so, when +the recorded spend reaches `--budget-usd`. + +Usage: + python benchmarks/independent/hunt.py --population population-v2.toml [--only ID,...] [--pins a,b] + python benchmarks/independent/hunt.py --population population-v2.toml --score + python benchmarks/independent/hunt.py --pilot-root --family injection # a development case +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import shutil +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import evaluate # noqa: E402 + +from openultrasast.mapping import analyze_entry_points # noqa: E402 +from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402 +from openultrasast.model.regions import regions_for # noqa: E402 +from openultrasast.model.shipped import declared_sources # noqa: E402 +from openultrasast.preprocess import build_file_target, enumerate_source_files # noqa: E402 + +# The model names a family in its own words; the protocol scores three. Fixed before any v2 pin is hunted. +FAMILY_WORDS = { + "untrusted_destination": ("ssrf", "server-side request", "open redirect", "redirect", "url fetch", "request forgery"), + "config_secrets": ("cors", "debug", "secret", "hardcoded", "hard-coded", "credential", "misconfig", "configuration", "tls", "certificate"), + "injection": ("inject", "sql", "sqli", "nosql", "command", "rce", "code execution", "eval", "template", "ssti", "xss", "deserializ", "ldap", "xpath"), +} + + +def family_of(raw: str, title: str) -> str: + text = f"{raw} {title}".lower() + for family, words in FAMILY_WORDS.items(): + if any(word in text for word in words): + return family + return "other" + + +def hotspots_for(root: Path, family: str, max_regions: int) -> list: + from openultrasast.complexity.map import Hotspot + + files = enumerate_source_files(root) + if not files or not sum(p.stat().st_size for p in files if p.is_file()): + raise SystemExit(f"read nothing under {root}") + targets = [build_file_target(root, p) for p in files] + regions = [ + dataclasses.replace(r, families=tuple(f for f in r.families if f == family)) + for r in regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root)) + ] + ranked = sorted((r for r in regions if r.families), key=lambda r: (-r.rank, r.path, r.function or ""))[:max_regions] + return [ + Hotspot( + path=r.path, + function_name=r.function or None, + score=float(r.rank), + band="ranker", + signals={}, + rationale=f"ranked region offering {family}", + test_hint=None, + inventory_finding_ids=(), + ) + for r in ranked + ] + + +def _spent(client: object) -> float: + cost = getattr(client, "cost_usd", None) + return float(cost()) if callable(cost) else 0.0 + + +def hunt(root: Path, family: str, args: argparse.Namespace, spent: list[float]) -> dict: + from openultrasast import tool_hunter + + client = tool_hunter.resolve_hunter_client() + if client is None: + raise SystemExit("no hunter client: set DEEPSEEK_API_KEY") + started = time.monotonic() + hotspots = hotspots_for(root, family, args.max_regions) + groups = [hotspots[i : i + args.per_hunt] for i in range(0, len(hotspots), args.per_hunt)] + findings, done, errors = [], 0, [] + for group in groups: + if sum(spent) + _spent(client) >= args.budget_usd: + errors.append("budget_exhausted") + break + try: + found = tool_hunter.run_tool_hunter(root, group, client=client, model=args.model, max_steps=tool_hunter.DEFAULT_MAX_STEPS) + done += 1 + except Exception as error: # noqa: BLE001 -- recorded per hunt; a pin with no completed hunt is an instrument failure + errors.append(f"{type(error).__name__}: {str(error)[:200]}") + continue + for f in found: + raw = next((t.split(":", 1)[1] for t in f.tags if t.startswith("family:")), "") + findings.append( + { + "site": f"{f.path}:{f.line or 0}:{f.function_name or ''}", + "family": family_of(raw, f.title), + "model_family": raw, + "title": f.title, + "witness": f.rationale[:400], + } + ) + cost = _spent(client) + spent.append(cost) + return { + "root": str(root), + "families": [family], + "model": args.model, + "seconds": round(time.monotonic() - started, 1), + "questions": len(groups), + "completed": done, + "completed_regions": sorted({f"{h.path}:{h.function_name or ''}" for g in groups[:done] for h in g}), + "hunt_errors": errors, + "cost_usd": round(cost, 4), + "findings": sorted(findings, key=lambda f: f["site"]), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--population", type=Path, default=evaluate.POPULATION) + parser.add_argument("--only", default="") + parser.add_argument("--pins", default="") + parser.add_argument("--pilot-root", type=Path) + parser.add_argument("--family", default="injection") + parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL) + parser.add_argument("--max-regions", type=int, default=20) + parser.add_argument("--per-hunt", type=int, default=5) + parser.add_argument("--budget-usd", type=float, default=10.0) + parser.add_argument("--score", action="store_true", help="score the hunter's results with the population's protocol") + args = parser.parse_args() + from openultrasast.config import load_config + + load_config() # loads the project's .env: DeepSeek is the chat provider, OpenRouter only embeddings + spent: list[float] = [] + if args.pilot_root: + result = hunt(args.pilot_root, args.family, args, spent) + print(json.dumps({k: v for k, v in result.items() if k != "findings"} | {"findings": result["findings"]}, indent=1)) + return 0 + evaluate.use(args.population) + evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-hunter") + if args.score: + return evaluate.score_v2() + only = {c.strip() for c in args.only.split(",") if c.strip()} + wanted = {p.strip() for p in args.pins.split(",") if p.strip()} + for case in evaluate.cases(): + if only and case["id"] not in only: + continue + for label, commit in evaluate.pins(case).items(): + if wanted and label not in wanted: + continue + out = evaluate.RESULTS / "scans" / f"{case['id']}--{label}.json" + if out.is_file(): + continue + if sum(spent) >= args.budget_usd: + print(json.dumps({"stopped": "budget", "spent_usd": round(sum(spent), 4)}), flush=True) + return 1 + out.parent.mkdir(parents=True, exist_ok=True) + checkout = evaluate.export(case, label, commit) + try: + result = hunt(checkout, case["family"], args, spent) + finally: + shutil.rmtree(checkout, ignore_errors=True) + if not result["completed"]: + result["instrument_error"] = "no hunt completed: " + "; ".join(result["hunt_errors"][:3]) + result["root"] = "/case" + out.write_text(json.dumps(result, indent=2) + "\n") + print( + json.dumps({"case": case["id"], "pin": label, "findings": len(result["findings"]), "hunts": f"{result['completed']}/{result['questions']}", "usd": result["cost_usd"], "total_usd": round(sum(spent), 4)}), + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/independent/hunter-protocol-v2.md b/benchmarks/independent/hunter-protocol-v2.md new file mode 100644 index 0000000..e933f91 --- /dev/null +++ b/benchmarks/independent/hunter-protocol-v2.md @@ -0,0 +1,55 @@ +# Hunter experiment on population v2 (pre-registered) + +Written and committed on 2026-09-28, BEFORE the tool hunter is run on any v2 pin. It is the deciding experiment +of the strategy review (`.kiro/specs/pre-push-safety-net/strategy-review-2026-09-28.md`): does a model with +repository tools find what the static engine did not (v2: recall 0/17, precision 0/17)? + +## What is under test + +- `tool_hunter.run_tool_hunter` as it stands: default system prompt, default tools (`read_file`, `grep_repo`, + `find_refs`), `DEFAULT_MAX_STEPS` (4), the final-answer turn. Its prompt, tools and parsing are unchanged since + the 2026-09-06 measurement; the only changes today are transport (cf57995 and this commit: the client reaches + DeepSeek, and the endpoint is priced so the budget can see spending). +- Model: `deepseek-flash` (DeepSeek, the project's chat provider), through `resolve_hunter_client()`. +- Runner: `benchmarks/independent/hunt.py` at the commit that introduces this file. + +## Scope handed to the model, per pin + +The regions the static scan's discovery and ranker produce for the case's family (`regions_for` over +`analyze_entry_points`, the same as `finding_dump.py`), sorted by rank, top 20, as hunter hotspots in four groups +of five: one bounded hunt per group. Nothing else: not the fix diff, not the advisory, not the case's declared +sites, not the case id. + +## Pins + +Protocol v2's six per case: vulnerable a/b, fixed a/b, benign base and tip. Each is a `git archive` of the pin. + +## Turning a hunter answer into a scoreable finding + +- `site` = `path:line:function`, with the function the model reported, or empty. +- `family` = the model's own family and title mapped by the fixed keyword table in `hunt.py` (`FAMILY_WORDS`, + checked in that order: untrusted_destination, config_secrets, injection; anything else is `other`). + +## Scoring + +Exactly protocol v2 (`protocol-v2.md`), via `evaluate.score_v2` over the hunter's results: the same matching +(v1's changed-line rule or a declared `path::function` site), detection in BOTH vulnerable runs, a fixed-side +alert in EITHER fixed run, benign alerts, stability, and the precision sample (every third stable +vulnerable-pin finding) adjudicated to v1's standard. A finding with no function can match only by the +changed-line rule. A pin whose four hunts all fail is an instrument failure, reported as such. + +"Questions" for the hunter are its hunts (4 per pin), so completion is hunts completed / hunts run; it is not +comparable to the engine's question completion and is reported only as an instrument check. + +## Cost + +Hard ceiling $10 of recorded model spend, enforced by the runner; a pilot on a v1 development case +(mongo-express, 4 hunts) cost $0.031 and 57 s. If the ceiling stops the run, the unscanned pins are reported +as not run. + +## What may not happen + +- No change to the hunter's prompt, tools, step budget, parsing, scope rule or family table before this result + is committed. +- Whatever is learned here is a finding about the hunter as it is; tuning it on v2 afterwards is a retune and + needs a new population to qualify. diff --git a/src/openultrasast/tool_hunter.py b/src/openultrasast/tool_hunter.py index 380fc9e..101e9a8 100644 --- a/src/openultrasast/tool_hunter.py +++ b/src/openultrasast/tool_hunter.py @@ -251,10 +251,11 @@ def resolve_hunter_client() -> ChatClient | None: if flag in _UNSAFE_FLAGS: return scripted_hunter_client("unsafe-snippet") if flag not in _OPENROUTER_FLAGS and os.environ.get("DEEPSEEK_API_KEY"): - from .model.endpoint import DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL, ChatEndpoint, DeepSeekChatClient + from .model.endpoint import DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL, DEFAULT_DETECTOR_MODEL, ChatEndpoint, DeepSeekChatClient, price_of base_url = os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL).rstrip("/") - endpoint = ChatEndpoint(provider="deepseek", base_url=base_url, thinking=False) + # Priced, so a budget can see what it spends: unpriced, the meter read $0.00 for any number of calls. + endpoint = ChatEndpoint(provider="deepseek", base_url=base_url, thinking=False, prices=price_of(DEFAULT_DETECTOR_MODEL)) return DeepSeekChatClient(OpenRouterChatClient(api_key=os.environ["DEEPSEEK_API_KEY"], base_url=base_url), endpoint=endpoint) if flag in _OPENROUTER_FLAGS or (not flag and os.environ.get("OPENROUTER_API_KEY")): try: diff --git a/tests/test_model_endpoint.py b/tests/test_model_endpoint.py index 249183b..308c8de 100644 --- a/tests/test_model_endpoint.py +++ b/tests/test_model_endpoint.py @@ -244,5 +244,7 @@ def test_the_hunter_reaches_deepseek_when_its_key_is_set(monkeypatch) -> None: monkeypatch.setenv("OPENROUTER_API_KEY", "embeddings-key") client = tool_hunter.resolve_hunter_client() assert isinstance(client, DeepSeekChatClient), type(client) + # Priced: a budget that reads $0.00 for any number of calls is not a budget. + assert client.endpoint is not None and client.endpoint.prices is not None monkeypatch.delenv("DEEPSEEK_API_KEY") assert not isinstance(tool_hunter.resolve_hunter_client(), DeepSeekChatClient)