mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
spec(pre-push-safety-net): pre-register the hunter experiment on population v2
This is the deciding experiment of the strategy review, fixed before any v2 pin is hunted. - Hunter: the unchanged tool hunter (default prompt, tools and 4 steps) on deepseek-flash. - Scope: the ranker's top 20 regions for the case family, in four hunts. The model never sees the fix diff or the declared sites. - Families: a fixed keyword table maps the model's family to the protocol's three. - Scoring: exactly protocol v2's scorer. - Cost: a hard $10 ceiling. hunt.py runs it and scores it (--score). A pilot on a v1 development case (mongo-express) found the vulnerability at lib/bson.js:60, at $0.031 per pin, alongside a lot of speculative noise. Transport fix found by the pilot: the DeepSeek endpoint the hunter builds was unpriced, so the cost meter read $0.00 for any number of calls. It is now priced. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
cf579956d5
commit
6bfb05f952
@@ -0,0 +1,191 @@
|
||||
"""Run the UNCHANGED tool hunter over an independent population's pins, scoreable by the evaluation protocol.
|
||||
|
||||
The hunter (`tool_hunter.run_tool_hunter`, default prompt, default tools, default step budget) is handed the
|
||||
regions the ranker selects for the case's family -- the same discovery and ranking the static scan starts from,
|
||||
top `--max-regions` by rank, in groups of `--per-hunt` -- and nothing else: not the fix diff, not the advisory,
|
||||
not the declared sites. Each pin's result is written in the scan-result shape `evaluate.py score` reads, into
|
||||
`~/ousast-results/independent-<vN>-hunter/scans/`, so the same pre-registered matching scores it.
|
||||
|
||||
Model calls go to the resolved hunter client (DeepSeek when its key is set); the run stops, and says so, when
|
||||
the recorded spend reaches `--budget-usd`.
|
||||
|
||||
Usage:
|
||||
python benchmarks/independent/hunt.py --population population-v2.toml [--only ID,...] [--pins a,b]
|
||||
python benchmarks/independent/hunt.py --population population-v2.toml --score
|
||||
python benchmarks/independent/hunt.py --pilot-root <checkout> --family injection # a development case
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import evaluate # noqa: E402
|
||||
|
||||
from openultrasast.mapping import analyze_entry_points # noqa: E402
|
||||
from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402
|
||||
from openultrasast.model.regions import regions_for # noqa: E402
|
||||
from openultrasast.model.shipped import declared_sources # noqa: E402
|
||||
from openultrasast.preprocess import build_file_target, enumerate_source_files # noqa: E402
|
||||
|
||||
# The model names a family in its own words; the protocol scores three. Fixed before any v2 pin is hunted.
|
||||
FAMILY_WORDS = {
|
||||
"untrusted_destination": ("ssrf", "server-side request", "open redirect", "redirect", "url fetch", "request forgery"),
|
||||
"config_secrets": ("cors", "debug", "secret", "hardcoded", "hard-coded", "credential", "misconfig", "configuration", "tls", "certificate"),
|
||||
"injection": ("inject", "sql", "sqli", "nosql", "command", "rce", "code execution", "eval", "template", "ssti", "xss", "deserializ", "ldap", "xpath"),
|
||||
}
|
||||
|
||||
|
||||
def family_of(raw: str, title: str) -> str:
|
||||
text = f"{raw} {title}".lower()
|
||||
for family, words in FAMILY_WORDS.items():
|
||||
if any(word in text for word in words):
|
||||
return family
|
||||
return "other"
|
||||
|
||||
|
||||
def hotspots_for(root: Path, family: str, max_regions: int) -> list:
|
||||
from openultrasast.complexity.map import Hotspot
|
||||
|
||||
files = enumerate_source_files(root)
|
||||
if not files or not sum(p.stat().st_size for p in files if p.is_file()):
|
||||
raise SystemExit(f"read nothing under {root}")
|
||||
targets = [build_file_target(root, p) for p in files]
|
||||
regions = [
|
||||
dataclasses.replace(r, families=tuple(f for f in r.families if f == family))
|
||||
for r in regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root))
|
||||
]
|
||||
ranked = sorted((r for r in regions if r.families), key=lambda r: (-r.rank, r.path, r.function or ""))[:max_regions]
|
||||
return [
|
||||
Hotspot(
|
||||
path=r.path,
|
||||
function_name=r.function or None,
|
||||
score=float(r.rank),
|
||||
band="ranker",
|
||||
signals={},
|
||||
rationale=f"ranked region offering {family}",
|
||||
test_hint=None,
|
||||
inventory_finding_ids=(),
|
||||
)
|
||||
for r in ranked
|
||||
]
|
||||
|
||||
|
||||
def _spent(client: object) -> float:
|
||||
cost = getattr(client, "cost_usd", None)
|
||||
return float(cost()) if callable(cost) else 0.0
|
||||
|
||||
|
||||
def hunt(root: Path, family: str, args: argparse.Namespace, spent: list[float]) -> dict:
|
||||
from openultrasast import tool_hunter
|
||||
|
||||
client = tool_hunter.resolve_hunter_client()
|
||||
if client is None:
|
||||
raise SystemExit("no hunter client: set DEEPSEEK_API_KEY")
|
||||
started = time.monotonic()
|
||||
hotspots = hotspots_for(root, family, args.max_regions)
|
||||
groups = [hotspots[i : i + args.per_hunt] for i in range(0, len(hotspots), args.per_hunt)]
|
||||
findings, done, errors = [], 0, []
|
||||
for group in groups:
|
||||
if sum(spent) + _spent(client) >= args.budget_usd:
|
||||
errors.append("budget_exhausted")
|
||||
break
|
||||
try:
|
||||
found = tool_hunter.run_tool_hunter(root, group, client=client, model=args.model, max_steps=tool_hunter.DEFAULT_MAX_STEPS)
|
||||
done += 1
|
||||
except Exception as error: # noqa: BLE001 -- recorded per hunt; a pin with no completed hunt is an instrument failure
|
||||
errors.append(f"{type(error).__name__}: {str(error)[:200]}")
|
||||
continue
|
||||
for f in found:
|
||||
raw = next((t.split(":", 1)[1] for t in f.tags if t.startswith("family:")), "")
|
||||
findings.append(
|
||||
{
|
||||
"site": f"{f.path}:{f.line or 0}:{f.function_name or ''}",
|
||||
"family": family_of(raw, f.title),
|
||||
"model_family": raw,
|
||||
"title": f.title,
|
||||
"witness": f.rationale[:400],
|
||||
}
|
||||
)
|
||||
cost = _spent(client)
|
||||
spent.append(cost)
|
||||
return {
|
||||
"root": str(root),
|
||||
"families": [family],
|
||||
"model": args.model,
|
||||
"seconds": round(time.monotonic() - started, 1),
|
||||
"questions": len(groups),
|
||||
"completed": done,
|
||||
"completed_regions": sorted({f"{h.path}:{h.function_name or ''}" for g in groups[:done] for h in g}),
|
||||
"hunt_errors": errors,
|
||||
"cost_usd": round(cost, 4),
|
||||
"findings": sorted(findings, key=lambda f: f["site"]),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--population", type=Path, default=evaluate.POPULATION)
|
||||
parser.add_argument("--only", default="")
|
||||
parser.add_argument("--pins", default="")
|
||||
parser.add_argument("--pilot-root", type=Path)
|
||||
parser.add_argument("--family", default="injection")
|
||||
parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
|
||||
parser.add_argument("--max-regions", type=int, default=20)
|
||||
parser.add_argument("--per-hunt", type=int, default=5)
|
||||
parser.add_argument("--budget-usd", type=float, default=10.0)
|
||||
parser.add_argument("--score", action="store_true", help="score the hunter's results with the population's protocol")
|
||||
args = parser.parse_args()
|
||||
from openultrasast.config import load_config
|
||||
|
||||
load_config() # loads the project's .env: DeepSeek is the chat provider, OpenRouter only embeddings
|
||||
spent: list[float] = []
|
||||
if args.pilot_root:
|
||||
result = hunt(args.pilot_root, args.family, args, spent)
|
||||
print(json.dumps({k: v for k, v in result.items() if k != "findings"} | {"findings": result["findings"]}, indent=1))
|
||||
return 0
|
||||
evaluate.use(args.population)
|
||||
evaluate.RESULTS = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-hunter")
|
||||
if args.score:
|
||||
return evaluate.score_v2()
|
||||
only = {c.strip() for c in args.only.split(",") if c.strip()}
|
||||
wanted = {p.strip() for p in args.pins.split(",") if p.strip()}
|
||||
for case in evaluate.cases():
|
||||
if only and case["id"] not in only:
|
||||
continue
|
||||
for label, commit in evaluate.pins(case).items():
|
||||
if wanted and label not in wanted:
|
||||
continue
|
||||
out = evaluate.RESULTS / "scans" / f"{case['id']}--{label}.json"
|
||||
if out.is_file():
|
||||
continue
|
||||
if sum(spent) >= args.budget_usd:
|
||||
print(json.dumps({"stopped": "budget", "spent_usd": round(sum(spent), 4)}), flush=True)
|
||||
return 1
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
checkout = evaluate.export(case, label, commit)
|
||||
try:
|
||||
result = hunt(checkout, case["family"], args, spent)
|
||||
finally:
|
||||
shutil.rmtree(checkout, ignore_errors=True)
|
||||
if not result["completed"]:
|
||||
result["instrument_error"] = "no hunt completed: " + "; ".join(result["hunt_errors"][:3])
|
||||
result["root"] = "/case"
|
||||
out.write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(
|
||||
json.dumps({"case": case["id"], "pin": label, "findings": len(result["findings"]), "hunts": f"{result['completed']}/{result['questions']}", "usd": result["cost_usd"], "total_usd": round(sum(spent), 4)}),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,55 @@
|
||||
# Hunter experiment on population v2 (pre-registered)
|
||||
|
||||
Written and committed on 2026-09-28, BEFORE the tool hunter is run on any v2 pin. It is the deciding experiment
|
||||
of the strategy review (`.kiro/specs/pre-push-safety-net/strategy-review-2026-09-28.md`): does a model with
|
||||
repository tools find what the static engine did not (v2: recall 0/17, precision 0/17)?
|
||||
|
||||
## What is under test
|
||||
|
||||
- `tool_hunter.run_tool_hunter` as it stands: default system prompt, default tools (`read_file`, `grep_repo`,
|
||||
`find_refs`), `DEFAULT_MAX_STEPS` (4), the final-answer turn. Its prompt, tools and parsing are unchanged since
|
||||
the 2026-09-06 measurement; the only changes today are transport (cf57995 and this commit: the client reaches
|
||||
DeepSeek, and the endpoint is priced so the budget can see spending).
|
||||
- Model: `deepseek-flash` (DeepSeek, the project's chat provider), through `resolve_hunter_client()`.
|
||||
- Runner: `benchmarks/independent/hunt.py` at the commit that introduces this file.
|
||||
|
||||
## Scope handed to the model, per pin
|
||||
|
||||
The regions the static scan's discovery and ranker produce for the case's family (`regions_for` over
|
||||
`analyze_entry_points`, the same as `finding_dump.py`), sorted by rank, top 20, as hunter hotspots in four groups
|
||||
of five: one bounded hunt per group. Nothing else: not the fix diff, not the advisory, not the case's declared
|
||||
sites, not the case id.
|
||||
|
||||
## Pins
|
||||
|
||||
Protocol v2's six per case: vulnerable a/b, fixed a/b, benign base and tip. Each is a `git archive` of the pin.
|
||||
|
||||
## Turning a hunter answer into a scoreable finding
|
||||
|
||||
- `site` = `path:line:function`, with the function the model reported, or empty.
|
||||
- `family` = the model's own family and title mapped by the fixed keyword table in `hunt.py` (`FAMILY_WORDS`,
|
||||
checked in that order: untrusted_destination, config_secrets, injection; anything else is `other`).
|
||||
|
||||
## Scoring
|
||||
|
||||
Exactly protocol v2 (`protocol-v2.md`), via `evaluate.score_v2` over the hunter's results: the same matching
|
||||
(v1's changed-line rule or a declared `path::function` site), detection in BOTH vulnerable runs, a fixed-side
|
||||
alert in EITHER fixed run, benign alerts, stability, and the precision sample (every third stable
|
||||
vulnerable-pin finding) adjudicated to v1's standard. A finding with no function can match only by the
|
||||
changed-line rule. A pin whose four hunts all fail is an instrument failure, reported as such.
|
||||
|
||||
"Questions" for the hunter are its hunts (4 per pin), so completion is hunts completed / hunts run; it is not
|
||||
comparable to the engine's question completion and is reported only as an instrument check.
|
||||
|
||||
## Cost
|
||||
|
||||
Hard ceiling $10 of recorded model spend, enforced by the runner; a pilot on a v1 development case
|
||||
(mongo-express, 4 hunts) cost $0.031 and 57 s. If the ceiling stops the run, the unscanned pins are reported
|
||||
as not run.
|
||||
|
||||
## What may not happen
|
||||
|
||||
- No change to the hunter's prompt, tools, step budget, parsing, scope rule or family table before this result
|
||||
is committed.
|
||||
- Whatever is learned here is a finding about the hunter as it is; tuning it on v2 afterwards is a retune and
|
||||
needs a new population to qualify.
|
||||
Reference in New Issue
Block a user