From d1aabf0f66cda1acc5e95f62940e6c33c8d7b7d4 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:09:28 +0200 Subject: [PATCH] ci: green again -- format and lint the whole tree; push the scan and runner images to GHCR on tags CI and the release job check `ruff format --check .` and `ruff check .` over the whole repository, while the local gate checked only src and tests, so eight benchmark scripts drifted and the v2.0.0 release job stopped before building. The fixes are formatting, two noqa comments and one renamed loop variable; no script computes anything differently. The local gate now checks the whole tree. The release workflow gains an images job: Dockerfile -> ghcr.io//openultrasast, plane/Dockerfile.runner -> ghcr.io//openultrasast-runner, tagged with the version, and the digests written to the job summary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .github/workflows/release.yml | 42 ++++++++ benchmarks/archive/hunt.py | 41 ++++++- benchmarks/archive/model_sinks.py | 18 +++- benchmarks/archive/score_batched.py | 3 +- benchmarks/archive/sink_candidates.py | 4 +- benchmarks/archive/verify_batched.py | 101 +++++++++++++++--- benchmarks/archive/verify_sinks.py | 60 +++++++++-- benchmarks/dev/bulk_read.py | 9 +- benchmarks/dev/read_guard.py | 6 +- benchmarks/independent/evaluate.py | 6 +- benchmarks/independent/freeze.py | 2 +- .../measure_php_quick.py | 2 +- 12 files changed, 256 insertions(+), 38 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bdb49c9..fef41d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,3 +34,45 @@ jobs: generate_release_notes: true prerelease: ${{ contains(github.ref_name, '-') }} make_latest: ${{ !contains(github.ref_name, '-') }} + + images: + # The scan image (Joern + php-cli, `docker run ... scan`) and the plane's task runner, pushed to GHCR under the + # tag's version. Pin them by the digests this job prints, not by tag: Agent Substrate rejects tags. + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - name: openultrasast + dockerfile: Dockerfile + - name: openultrasast-runner + dockerfile: plane/Dockerfile.runner + steps: + - uses: actions/checkout@v5 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref_name, '-') }} + type=raw,value=latest,enable=${{ !contains(github.ref_name, '-') }} + - id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - run: | + echo "ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_STEP_SUMMARY" diff --git a/benchmarks/archive/hunt.py b/benchmarks/archive/hunt.py index b65aeae..b9a0d42 100644 --- a/benchmarks/archive/hunt.py +++ b/benchmarks/archive/hunt.py @@ -40,8 +40,34 @@ from openultrasast.preprocess import build_file_target, enumerate_source_files # The model names a family in its own words; the protocol scores three. Fixed before any v2 pin is hunted. FAMILY_WORDS = { "untrusted_destination": ("ssrf", "server-side request", "open redirect", "redirect", "url fetch", "request forgery"), - "config_secrets": ("cors", "debug", "secret", "hardcoded", "hard-coded", "credential", "misconfig", "configuration", "tls", "certificate"), - "injection": ("inject", "sql", "sqli", "nosql", "command", "rce", "code execution", "eval", "template", "ssti", "xss", "deserializ", "ldap", "xpath"), + "config_secrets": ( + "cors", + "debug", + "secret", + "hardcoded", + "hard-coded", + "credential", + "misconfig", + "configuration", + "tls", + "certificate", + ), + "injection": ( + "inject", + "sql", + "sqli", + "nosql", + "command", + "rce", + "code execution", + "eval", + "template", + "ssti", + "xss", + "deserializ", + "ldap", + "xpath", + ), } @@ -182,7 +208,16 @@ def main() -> int: result["root"] = "/case" out.write_text(json.dumps(result, indent=2) + "\n") print( - json.dumps({"case": case["id"], "pin": label, "findings": len(result["findings"]), "hunts": f"{result['completed']}/{result['questions']}", "usd": result["cost_usd"], "total_usd": round(sum(spent), 4)}), + json.dumps( + { + "case": case["id"], + "pin": label, + "findings": len(result["findings"]), + "hunts": f"{result['completed']}/{result['questions']}", + "usd": result["cost_usd"], + "total_usd": round(sum(spent), 4), + } + ), flush=True, ) return 0 diff --git a/benchmarks/archive/model_sinks.py b/benchmarks/archive/model_sinks.py index 6860e35..eec528b 100644 --- a/benchmarks/archive/model_sinks.py +++ b/benchmarks/archive/model_sinks.py @@ -37,7 +37,9 @@ from openultrasast.config import load_config # noqa: E402 from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL # noqa: E402 from openultrasast.preprocess import detect_language, enumerate_source_files # noqa: E402 -NOT_PRODUCT = re.compile(r"(^|/)(tests?|__tests__|spec|specs|docs?|examples?|fixtures?|e2e|cypress|storybook)/|\.(test|spec|stories)\.[jt]sx?$|_test\.py$|(^|/)test_[^/]*\.py$|\.min\.js$|(^|/)dist/|(^|/)build/") +NOT_PRODUCT = re.compile( + r"(^|/)(tests?|__tests__|spec|specs|docs?|examples?|fixtures?|e2e|cypress|storybook)/|\.(test|spec|stories)\.[jt]sx?$|_test\.py$|(^|/)test_[^/]*\.py$|\.min\.js$|(^|/)dist/|(^|/)build/" +) OPERATIONS = { "injection": ( "executes a database query (SQL, NoSQL, JCR/graph query), an operating-system command or shell, code or " @@ -69,7 +71,9 @@ def product_files(root: Path, languages: set[str]) -> list[Path]: def chunks(text: str, size: int) -> list[str]: lines = text.splitlines() - return ["\n".join(f"{n + 1:5d}| {line}" for n, line in enumerate(lines[i : i + size], start=i)) for i in range(0, max(len(lines), 1), size)] + return [ + "\n".join(f"{n + 1:5d}| {line}" for n, line in enumerate(lines[i : i + size], start=i)) for i in range(0, max(len(lines), 1), size) + ] class Meter: @@ -85,7 +89,9 @@ class Meter: return sum(float(c.cost_usd()) for c in self.clients) -def classify(root: Path, path: Path, family: str, args: argparse.Namespace, meter: Meter, local: threading.local) -> tuple[list[tuple[str, str, int]], int, int]: +def classify( + root: Path, path: Path, family: str, args: argparse.Namespace, meter: Meter, local: threading.local +) -> tuple[list[tuple[str, str, int]], int, int]: from openultrasast import tool_hunter if not hasattr(local, "client"): @@ -99,7 +105,9 @@ def classify(root: Path, path: Path, family: str, args: argparse.Namespace, mete asked += 1 prompt = PROMPT.format(operation=OPERATIONS[family], path=relative, code=code) try: - response = local.client.complete(model=args.model, messages=[{"role": "user", "content": prompt}], tools=[], timeout_seconds=90, json_object=True) + response = local.client.complete( + model=args.model, messages=[{"role": "user", "content": prompt}], tools=[], timeout_seconds=90, json_object=True + ) items = json.loads(response.content or "{}").get("functions", []) except Exception: # noqa: BLE001 -- counted as unclassified, never as "no sinks" failed += 1 @@ -138,7 +146,7 @@ def main() -> int: files = product_files(checkout, {case["language"], *case.get("also", [])}) found, failed, asked = [], 0, 0 with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool: - for f, fail, ask in pool.map(lambda p: classify(checkout, p, case["family"], args, meter, local), files): + for f, fail, ask in pool.map(lambda p, c=checkout, fam=case["family"]: classify(c, p, fam, args, meter, local), files): found += f failed += fail asked += ask diff --git a/benchmarks/archive/score_batched.py b/benchmarks/archive/score_batched.py index b26b3b0..593715f 100644 --- a/benchmarks/archive/score_batched.py +++ b/benchmarks/archive/score_batched.py @@ -66,7 +66,8 @@ def main() -> int: "agreed_findings": len(vulnerable["findings"]), "disputed_findings": len(vulnerable.get("disputed", [])), "triaged_out": vulnerable.get("triaged_out"), "candidates": vulnerable.get("questions"), "fixed_side_alerts": fixed_alerts, "fixed_agreed_total": len(fixed["findings"]) if fixed else None, - "removed_by_fix": fixed.get("removed_by_fix") if fixed else None, "usd": vulnerable.get("usd"), "usage": vulnerable.get("usage"), + "removed_by_fix": fixed.get("removed_by_fix") if fixed else None, + "usd": vulnerable.get("usd"), "usage": vulnerable.get("usage"), } ) # fmt: skip pool += [{"case": case["id"], **f} for f in vulnerable["findings"]] diff --git a/benchmarks/archive/sink_candidates.py b/benchmarks/archive/sink_candidates.py index 051a1a7..19d77ef 100644 --- a/benchmarks/archive/sink_candidates.py +++ b/benchmarks/archive/sink_candidates.py @@ -36,7 +36,9 @@ from openultrasast.preprocess import detect_language, enumerate_source_files # DECLARATION = { "python": re.compile(r"^\s*(?:async\s+)?def\s+(\w+)"), "php": re.compile(r"^\s*(?:(?:public|private|protected|static|final|abstract)\s+)*function\s+&?(\w+)"), - "javascript": re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s*\*?\s*(\w+)|^\s*(?:(?:public|private|protected|static|async)\s+)*(\w+)\s*\([^;]*\)\s*(?::[^{=]+)?\{\s*$|^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:function|\()"), + "javascript": re.compile( + r"^\s*(?:export\s+)?(?:async\s+)?function\s*\*?\s*(\w+)|^\s*(?:(?:public|private|protected|static|async)\s+)*(\w+)\s*\([^;]*\)\s*(?::[^{=]+)?\{\s*$|^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:function|\()" + ), } DECLARATION["typescript"] = DECLARATION["javascript"] KEYWORDS = {"if", "for", "while", "switch", "catch", "return", "function", "else"} diff --git a/benchmarks/archive/verify_batched.py b/benchmarks/archive/verify_batched.py index 4bef059..d02dcd1 100644 --- a/benchmarks/archive/verify_batched.py +++ b/benchmarks/archive/verify_batched.py @@ -101,13 +101,22 @@ def triage(root: Path, path: str, candidates: list[list], family: str, model: st except OSError: return {} names = sorted({fn for _, fn, _ in candidates}) - prompt = TRIAGE.format(operation=OPERATIONS[family], functions=json.dumps(names), path=path, code=_relevant_code(text, [ln for _, _, ln in candidates], chunk_lines)) + prompt = TRIAGE.format( + operation=OPERATIONS[family], + functions=json.dumps(names), + path=path, + code=_relevant_code(text, [ln for _, _, ln in candidates], chunk_lines), + ) response = client.complete(model=model, messages=[{"role": "user", "content": prompt}], tools=[], timeout_seconds=90, json_object=True) try: items = json.loads(response.content or "{}").get("functions", []) except (json.JSONDecodeError, AttributeError): return {} - return {str(i.get("function")): str(i.get("data_origin", "")).strip().lower() for i in items if isinstance(i, dict) and i.get("function") in names} + return { + str(i.get("function")): str(i.get("data_origin", "")).strip().lower() + for i in items + if isinstance(i, dict) and i.get("function") in names + } def hunt(root: Path, path: str, group: list[list], family: str, model: str, steps: int) -> tuple[list[dict], dict]: @@ -120,8 +129,30 @@ def hunt(root: Path, path: str, group: list[list], family: str, model: str, step f"Operations to judge, all in `{path}`, each of which {OPERATIONS[family]}:\n{listing}\n" "The file is above. For each, trace where the operation's data comes from and whether it is guarded." ) - spots = [Hotspot(path=path, function_name=fn, score=1.0, band="candidate", signals={}, rationale="model-classified sink", test_hint=None, inventory_finding_ids=()) for _, fn, _ in group] - found = tool_hunter.run_tool_hunter(root, spots, client=recording, model=model, max_steps=steps, system_prompt=SYSTEM, user_prompt=prompt, context_files=[path], tags=("sink-verifier",)) + spots = [ + Hotspot( + path=path, + function_name=fn, + score=1.0, + band="candidate", + signals={}, + rationale="model-classified sink", + test_hint=None, + inventory_finding_ids=(), + ) + for _, fn, _ in group + ] + found = tool_hunter.run_tool_hunter( + root, + spots, + client=recording, + model=model, + max_steps=steps, + system_prompt=SYSTEM, + user_prompt=prompt, + context_files=[path], + tags=("sink-verifier",), + ) names = {fn for _, fn, _ in group} rows = [] for f in found: @@ -131,7 +162,16 @@ def hunt(root: Path, path: str, group: list[list], family: str, model: str, step # The verdict is about the candidate operation, so the site is the candidate's file and function; the # model's own location is kept beside it (it often names the file where the input enters instead). line = f.line if f.path == path else next(c[2] for c in group if c[1] == fn) - rows.append({"candidate": f"{path}::{fn}", "site": f"{path}:{line or 0}:{fn}", "reported_at": f"{f.path}:{f.line or 0}", "family": family, "title": f.title, "witness": f.rationale[:600]}) + rows.append( + { + "candidate": f"{path}::{fn}", + "site": f"{path}:{line or 0}:{fn}", + "reported_at": f"{f.path}:{f.line or 0}", + "family": family, + "title": f.title, + "witness": f.rationale[:600], + } + ) return rows, recording.usage() @@ -143,7 +183,12 @@ def verify_file(root: str, path: str, candidates: list[list], family: str, args_ try: if not args_["no_triage"]: record["triage"] = triage(Path(root), path, candidates, family, args_["model"], args_["chunk_lines"]) - kept = [c for c in candidates if record["triage"].get(c[1], "unclassified") in VERIFY | {"unclassified"} or record["triage"].get(c[1]) not in {"internal", "constant"}] + kept = [ + c + for c in candidates + if record["triage"].get(c[1], "unclassified") in VERIFY | {"unclassified"} + or record["triage"].get(c[1]) not in {"internal", "constant"} + ] record["kept"] = [c[1] for c in kept] usage: dict[str, int] = {} for _pass in range(args_["passes"]): @@ -169,14 +214,16 @@ def agreement(record: dict) -> tuple[list[dict], list[dict]]: some = set.union(*(set(b) for b in by_pass)) first = by_pass[0] pick = lambda c: next(b[c] for b in by_pass if c in b) # noqa: E731 - return [dict(first[c], passes=len(by_pass)) for c in sorted(every)], [dict(pick(c), passes=sum(c in b for b in by_pass)) for c in sorted(some - every)] + return [dict(first[c], passes=len(by_pass)) for c in sorted(every)], [ + dict(pick(c), passes=sum(c in b for b in by_pass)) for c in sorted(some - every) + ] def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml") parser.add_argument("--only", default="") - parser.add_argument("--subset", type=Path, help='JSON {case_id: [[path, function], ...]}: verify only these candidates') + parser.add_argument("--subset", type=Path, help="JSON {case_id: [[path, function], ...]}: verify only these candidates") parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL) parser.add_argument("--max-steps", type=int, default=6) parser.add_argument("--per-hunt", type=int, default=6) @@ -206,7 +253,11 @@ def main() -> int: if out.is_file(): continue log = out_dir / f"{case['id']}--{label}.jsonl" - done = {r["path"]: r for r in (json.loads(row) for row in log.read_text().splitlines()) if "unverified" not in r} if log.is_file() else {} + done = ( + {r["path"]: r for r in (json.loads(row) for row in log.read_text().splitlines()) if "unverified" not in r} + if log.is_file() + else {} + ) spent += sum(float(r.get("usd", 0.0)) for r in done.values()) record = json.loads((candidates_dir / f"{case['id']}.json").read_text()) unique = list({(p, fn): [p, fn, ln] for p, fn, ln in sorted(record["candidates"])}.values()) @@ -265,19 +316,41 @@ def main() -> int: agreed += a disputed += d triaged_out = sum(len(r["candidates"]) - len(r.get("kept", [])) for r in done.values()) - usage = {k: sum(int(r.get("usage", {}).get(k, 0)) for r in done.values()) for k in ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens", "calls")} + usage = { + k: sum(int(r.get("usage", {}).get(k, 0)) for r in done.values()) + for k in ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens", "calls") + } complete = len(done) >= len(by_file) and not account_error result = { - "root": "/case", "families": [case["family"]], "questions": len(unique), "completed": sum(len(r["candidates"]) for r in done.values()), + "root": "/case", "families": [case["family"]], + "questions": len(unique), "completed": sum(len(r["candidates"]) for r in done.values()), "completed_regions": sorted(f"{c[0]}:{c[1]}" for r in done.values() for c in r["candidates"]), - "triaged_out": triaged_out, "passes": args.passes, "usage": usage, "usd": round(sum(float(r.get("usd", 0.0)) for r in done.values()), 4), - "seconds": round(time.monotonic() - started, 1), "findings": sorted(agreed, key=lambda f: f["site"]), "disputed": sorted(disputed, key=lambda f: f["site"]), + "triaged_out": triaged_out, "passes": args.passes, "usage": usage, + "usd": round(sum(float(r.get("usd", 0.0)) for r in done.values()), 4), + "seconds": round(time.monotonic() - started, 1), + "findings": sorted(agreed, key=lambda f: f["site"]), + "disputed": sorted(disputed, key=lambda f: f["site"]), } # fmt: skip if args.fixed: result["removed_by_fix"] = removed if complete: out.write_text(json.dumps(result, indent=1) + "\n") - print(json.dumps({"case": case["id"], "files": f"{len(done)}/{len(by_file)}", "candidates": len(unique), "triaged_out": triaged_out, "agreed": len(agreed), "disputed": len(disputed), "usd": result["usd"], "total_usd": round(spent, 4), "complete": complete}), flush=True) + print( + json.dumps( + { + "case": case["id"], + "files": f"{len(done)}/{len(by_file)}", + "candidates": len(unique), + "triaged_out": triaged_out, + "agreed": len(agreed), + "disputed": len(disputed), + "usd": result["usd"], + "total_usd": round(spent, 4), + "complete": complete, + } + ), + flush=True, + ) if account_error: print(json.dumps({"stopped": "account", "error": account_error}), flush=True) return 2 diff --git a/benchmarks/archive/verify_sinks.py b/benchmarks/archive/verify_sinks.py index 8d61354..42e72b3 100644 --- a/benchmarks/archive/verify_sinks.py +++ b/benchmarks/archive/verify_sinks.py @@ -104,7 +104,16 @@ def verify(root: str, candidate: list, family: str, model: str, max_steps: int, client = _client() before = float(client.cost_usd()) path, function, line = candidate - spot = Hotspot(path=path, function_name=function, score=1.0, band="candidate", signals={}, rationale="model-classified sink", test_hint=None, inventory_finding_ids=()) + spot = Hotspot( + path=path, + function_name=function, + score=1.0, + band="candidate", + signals={}, + rationale="model-classified sink", + test_hint=None, + inventory_finding_ids=(), + ) prompt = ( f"Operation to judge: in `{path}`, function `{function}`, around line {line}, code that {OPERATIONS[family]}. " "The file is above. Trace where the operation's data comes from and whether it is guarded." @@ -120,9 +129,19 @@ def verify(root: str, candidate: list, family: str, model: str, max_steps: int, second = recording.usage() usage = {k: usage.get(k, 0) + second.get(k, 0) for k in set(usage) | set(second)} except Exception as error: # noqa: BLE001 -- recorded; an unverified candidate is not a clean one - return {"candidate": candidate, "unverified": f"{type(error).__name__}: {str(error)[:120]}", "usd": float(client.cost_usd()) - before} + return { + "candidate": candidate, + "unverified": f"{type(error).__name__}: {str(error)[:120]}", + "usd": float(client.cost_usd()) - before, + } findings = [ - {"site": f"{f.path}:{f.line or line}:{f.function_name or function}", "family": family, "title": f.title, "witness": f.rationale[:600], "candidate": f"{path}::{function}"} + { + "site": f"{f.path}:{f.line or line}:{f.function_name or function}", + "family": family, + "title": f.title, + "witness": f.rationale[:600], + "candidate": f"{path}::{function}", + } for f in found ] return {"candidate": candidate, "findings": findings, "usd": float(client.cost_usd()) - before, "escalated": escalated, "usage": usage} @@ -134,7 +153,12 @@ def main() -> int: parser.add_argument("--only", default="") parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL) parser.add_argument("--max-steps", type=int, default=6) - parser.add_argument("--escalate-steps", type=int, default=0, help="rerun a hunt that used every step without a verdict; 0 disables (measured: 88% escalate, +45% cost)") + parser.add_argument( + "--escalate-steps", + type=int, + default=0, + help="rerun a hunt that used every step without a verdict; 0 disables (measured: 88% escalate, +45% cost)", + ) parser.add_argument("--workers", type=int, default=4) parser.add_argument("--budget-usd", type=float, default=40.0) parser.add_argument("--fixed", action="store_true", help="recheck detected sites at the fixed pin") @@ -185,7 +209,9 @@ def main() -> int: with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as pool, log.open("a") as sink: futures = [] for candidate in pending: - futures.append(pool.submit(verify, str(checkout), list(candidate), case["family"], args.model, args.max_steps, args.escalate_steps)) + futures.append( + pool.submit(verify, str(checkout), list(candidate), case["family"], args.model, args.max_steps, args.escalate_steps) + ) for future in concurrent.futures.as_completed(futures): if future.cancelled(): # cancelled at the ceiling: never run, nothing to record continue @@ -219,9 +245,29 @@ def main() -> int: } # fmt: skip if complete: out.write_text(json.dumps(result, indent=1) + "\n") - tokens = {k: sum(int(d.get("usage", {}).get(k, 0)) for d in done) for k in ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens")} + tokens = { + k: sum(int(d.get("usage", {}).get(k, 0)) for d in done) + for k in ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens") + } result["usage"] = tokens | {"escalated": sum(1 for d in done if d.get("escalated"))} - print(json.dumps({"case": case["id"], "pin": label, "candidates": len(todo), "verified": len(verified), "findings": len(findings), "unverified": len(unverified), "escalated": result["usage"]["escalated"], "cache_share": round(tokens["prompt_cache_hit_tokens"] / max(tokens["prompt_tokens"], 1), 2), "usd": result["usd"], "total_usd": round(spent, 4), "complete": complete}), flush=True) + print( + json.dumps( + { + "case": case["id"], + "pin": label, + "candidates": len(todo), + "verified": len(verified), + "findings": len(findings), + "unverified": len(unverified), + "escalated": result["usage"]["escalated"], + "cache_share": round(tokens["prompt_cache_hit_tokens"] / max(tokens["prompt_tokens"], 1), 2), + "usd": result["usd"], + "total_usd": round(spent, 4), + "complete": complete, + } + ), + flush=True, + ) if account_error: print(json.dumps({"stopped": "account", "error": account_error, "spent_usd": round(spent, 4)}), flush=True) return 2 diff --git a/benchmarks/dev/bulk_read.py b/benchmarks/dev/bulk_read.py index 04c3157..5349045 100644 --- a/benchmarks/dev/bulk_read.py +++ b/benchmarks/dev/bulk_read.py @@ -36,9 +36,9 @@ def main() -> int: parser.add_argument("files", nargs="+", type=Path) parser.add_argument("--max-usd", type=float, default=0.05) args = parser.parse_args() + from openultrasast import tool_hunter from openultrasast.config import load_config from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL - from openultrasast import tool_hunter load_config() client = tool_hunter.resolve_hunter_client() @@ -51,7 +51,12 @@ def main() -> int: if float(client.cost_usd()) >= args.max_usd: answers.append(f"[stopped at the ${args.max_usd} ceiling before: {chunk.splitlines()[0]}]") break - response = client.complete(model=DEFAULT_DETECTOR_MODEL, messages=[{"role": "user", "content": PROMPT.format(question=args.question, files=chunk)}], tools=[], timeout_seconds=90) + response = client.complete( + model=DEFAULT_DETECTOR_MODEL, + messages=[{"role": "user", "content": PROMPT.format(question=args.question, files=chunk)}], + tools=[], + timeout_seconds=90, + ) answers.append((response.content or "").strip()) print("\n\n".join(a for a in answers if a)) print(f"\n[bulk_read: {len(chunks)} chunk(s), ${float(client.cost_usd()):.4f}]", file=sys.stderr) diff --git a/benchmarks/dev/read_guard.py b/benchmarks/dev/read_guard.py index c6fa3cc..bf1df8f 100644 --- a/benchmarks/dev/read_guard.py +++ b/benchmarks/dev/read_guard.py @@ -37,7 +37,11 @@ def main() -> int: f"{path} has {lines} lines (limit {LIMIT} for a whole-file read). Read a range with offset/limit, or ask " f"the bulk reader for what you need: .venv/bin/python benchmarks/dev/bulk_read.py '' {path}" ) - print(json.dumps({"hookSpecificOutput": {"hookEventName": "PreToolUse", "permissionDecision": "deny", "permissionDecisionReason": reason}})) + print( + json.dumps( + {"hookSpecificOutput": {"hookEventName": "PreToolUse", "permissionDecision": "deny", "permissionDecisionReason": reason}} + ) + ) return 0 diff --git a/benchmarks/independent/evaluate.py b/benchmarks/independent/evaluate.py index 7fa7463..099c28b 100644 --- a/benchmarks/independent/evaluate.py +++ b/benchmarks/independent/evaluate.py @@ -258,7 +258,7 @@ def score_v2() -> int: per_case, pool, unstable_pool = [], [], 0 changing, total_questions, total_completed = [0, 0], 0, 0 for case in cases(): - path = lambda label: RESULTS / "scans" / f"{case['id']}--{label}.json" # noqa: E731 + path = lambda label: RESULTS / "scans" / f"{case['id']}--{label}.json" # noqa: E731, B023 -- called only inside this iteration scans = {label: json.loads(path(label).read_text()) for label in pins(case) if path(label).is_file()} if len(scans) < 6: per_case.append({"id": case["id"], "status": "incomplete", "scans": sorted(scans)}) @@ -279,7 +279,9 @@ def score_v2() -> int: ] site_files = {site.partition("::")[0] for site in case.get("sites", [])} covered = set(old) | site_files - answered = any(r.split(":")[0] in covered for label in ("vulnerable_a", "vulnerable_b") for r in scans[label].get("completed_regions", [])) + answered = any( + r.split(":")[0] in covered for label in ("vulnerable_a", "vulnerable_b") for r in scans[label].get("completed_regions", []) + ) diff_v, union_v = stability(va, vb) diff_f, union_f = stability(fa, fb) changing[0] += diff_v + diff_f diff --git a/benchmarks/independent/freeze.py b/benchmarks/independent/freeze.py index 038bbd0..876646c 100644 --- a/benchmarks/independent/freeze.py +++ b/benchmarks/independent/freeze.py @@ -44,7 +44,7 @@ EXTENSIONS = { # scopes to API tokens" -- access control and input validation, which is exactly what a benign control must not be. SECURITY_WORDS = re.compile( r"secur|cve|ghsa|vuln|inject|xss|csrf|ssrf|sanitiz|escap|exploit|attack|auth|permission|cors|access|restrict|harden" - r"|token|scope|valid|filter|privileg|role|owner|password|secret|redirect|sql|query|login|log in|ldap|session|ssl|tls|verif|cert|crypt|hash|cookie", + r"|token|scope|valid|filter|privileg|role|owner|password|secret|redirect|sql|query|login|log in|ldap|session|ssl|tls|verif|cert|crypt|hash|cookie", # noqa: E501 re.I, ) MAX_BENIGN_LINES = 200 diff --git a/benchmarks/measurements/2026-09-30-php-quick-rules/measure_php_quick.py b/benchmarks/measurements/2026-09-30-php-quick-rules/measure_php_quick.py index 8688e65..12d5332 100644 --- a/benchmarks/measurements/2026-09-30-php-quick-rules/measure_php_quick.py +++ b/benchmarks/measurements/2026-09-30-php-quick-rules/measure_php_quick.py @@ -162,7 +162,7 @@ def per_rule(result: dict) -> dict: rows[fid.split(":", 1)[0]][label] += 1 for fid in pin["at_known_site"]: rows[fid.split(":", 1)[0]][label + "_at_site"] += 1 - for name, fixture in result["fixtures"].items(): + for _name, fixture in result["fixtures"].items(): for fid in fixture["true_positives"]: rows[fid.split(":", 1)[0]]["fixture_tp"] += 1 for fid in fixture["false_positives"]: