chore: format the whole tree and lint the scratch measurement scripts; CI has been red since 25ae4b4

CI runs ruff format --check and ruff check over the whole repository; 29 files had drifted from the formatter and
four measurement scripts carried lint findings, so every push since 25ae4b4 failed before pytest ran.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/measurements/2026-10-12-banks-fixed-search-mvp/banks_mvp2.py

@@ -2,20 +2,25 @@
 taint.sc exec-signature skew). Build banks CPG, run the taint BATCH on the vuln functions with the annotation
 OFF (shipped) vs ON (parameter_sources=True + write_text/read_text added to path sinks). A non-empty flow row =
 the engine traced source->sink. Thesis: same target, only the search changes."""
-import dataclasses, json, sys
+
+import dataclasses
+import sys
 from pathlib import Path
+
 sys.path.insert(0, "/venv/lib/python3.12/site-packages")
 from openultrasast.cpg.backend import resolve_cpg_backend
 from openultrasast.model.specs import taint_specs
 from openultrasast.model.taint import request_params
 
-ROOT = Path("/work/banks"); VP = "src/banks/registries/directory.py"
+ROOT = Path("/work/banks")
+VP = "src/banks/registries/directory.py"
 FUNCS = ["from_prompt_path", "set", "_load", "_scan"]
 
 backend = resolve_cpg_backend()
 cpg = backend.build(ROOT, language="python")
 if cpg is None or not callable(getattr(cpg, "run_batch", None)):
-    print("CPG build failed or no run_batch"); raise SystemExit(1)
+    print("CPG build failed or no run_batch")
+    raise SystemExit(1)
 print("CPG built; run_batch available.", flush=True)
 
 base = taint_specs(language="python")["path"]
@@ -23,20 +28,22 @@ annot = dataclasses.replace(base, sinks=tuple(sorted(set(base.sinks) | {"write_t
 print("baseline sinks:", base.sinks)
 print("annotated sinks:", annot.sinks, flush=True)
 
+
 def run(label, spec, fn, psrc):
     reqs = {"r1": request_params(spec, function=fn, file=VP, parameter_sources=psrc)}
     try:
         out = cpg.run_batch("taint", reqs)
     except Exception as e:
-        print(f"  [{label}] {fn}: ERROR {type(e).__name__}: {str(e)[:150]}"); return 0
+        print(f"  [{label}] {fn}: ERROR {type(e).__name__}: {str(e)[:150]}")
+        return 0
     rows = (out or {}).get("r1") or []
-    n = len([r for r in rows if isinstance(r, (dict,)) ])
     print(f"  [{label}] {fn} psrc={psrc}: {len(rows)} flow row(s)")
     for r in rows[:3]:
         if isinstance(r, dict):
             print(f"      sink={r.get('sink')} source={r.get('source')} line={r.get('line')} evidence={str(r.get('evidence'))[:60]}")
     return len(rows)
 
+
 print("\n=== BASELINE (shipped search) ===", flush=True)
 b = sum(run("baseline", base, fn, False) for fn in FUNCS)
 print("\n=== FIXED SEARCH (annotation: param-source + write_text/read_text sink) ===", flush=True)

benchmarks/measurements/2026-10-12-valid-benchmark-pilot/filter_sweep.py

@@ -4,6 +4,7 @@ For every hunter defect: classifier verdict, confidence, taxonomy family, and wh
 location sits on a line the push changed (git diff base..head, head side). Then score each
 filter with the real scorer: recall over the 12 positives, false alarms per 100 ordinary pushes.
 """
+
 import copy
 import json
 import re
@@ -16,7 +17,7 @@ sys.path.insert(0, "/home/mc/Source/OpenUltraSAST")
 from benchmarks.unseen import score  # noqa: E402
 
 P = Path("/home/mc/ousast-results/valid/pilot")
-repos = json.load(open(P / "repos_hist.json"))
+repos = json.loads((P / "repos_hist.json").read_text())
 rows = score.load_records(P / "hunter")
 HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
 
@@ -24,7 +25,9 @@ HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
 def changed_lines(repo, base, head):
     out = subprocess.run(
         ["git", "-C", repos[repo], "diff", "-U0", f"{base}..{head}", "--", "."],
-        capture_output=True, text=True, check=False,
+        capture_output=True,
+        text=True,
+        check=False,
     ).stdout
     lines, path = {}, None
     for line in out.splitlines():
@@ -84,12 +87,14 @@ filters = {
     "vulnerable, within 10 of a changed line": lambda d: V(d) == "vulnerable" and d["_changed10"],
     "vulnerable, conf>=0.7, within 10 of changed": lambda d: V(d) == "vulnerable" and conf(d) >= 0.7 and d["_changed10"],
     "vulnerable, conf>=0.8, within 10 of changed": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"],
-    "vulnerable, conf>=0.8, within 10, not config_secrets": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"] and d.get("family") != "config_secrets",
+    "vulnerable, conf>=0.8, within 10, not config_secrets": lambda d: (
+        V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"] and d.get("family") != "config_secrets"
+    ),
 }
 print(f"{'filter':58} recall     false alarms /100   flagged pushes")
 for name, keep in filters.items():
     c, n, a, m, f = rate(rows, keep)
-    print(f"{name:58} {c:2}/{n:<2}      {a:3}/{m} = {100*a/m:5.1f}      {f}")
+    print(f"{name:58} {c:2}/{n:<2}      {a:3}/{m} = {100 * a / m:5.1f}      {f}")
 
 # where do ordinary-push suspicions sit relative to the diff?
 tot = Counter()
@@ -103,7 +108,19 @@ for r in rows:
             tot["within 10"] += d["_changed10"]
             tot[f"family {d.get('family')}"] += 1
 print("\nordinary-push suspicions with verdict=vulnerable:", dict(tot))
-confs = Counter(round(conf(d), 2) for r in rows if r["kind"] == "ordinary" for d in r.get("push", {}).get("admission", {}).get("defects", []) if V(d) == "vulnerable")
+confs = Counter(
+    round(conf(d), 2)
+    for r in rows
+    if r["kind"] == "ordinary"
+    for d in r.get("push", {}).get("admission", {}).get("defects", [])
+    if V(d) == "vulnerable"
+)
 print("confidence histogram (ordinary, vulnerable):", dict(sorted(confs.items())))
-confs = Counter(round(conf(d), 2) for r in rows if r["kind"] != "ordinary" for d in r.get("push", {}).get("admission", {}).get("defects", []) if V(d) == "vulnerable" and score.catches(filtered(r, lambda x, d=d: x is d)))
+confs = Counter(
+    round(conf(d), 2)
+    for r in rows
+    if r["kind"] != "ordinary"
+    for d in r.get("push", {}).get("admission", {}).get("defects", [])
+    if V(d) == "vulnerable" and score.catches(filtered(r, lambda x, d=d: x is d))
+)
 print("confidence of the defects that score a catch:", dict(sorted(confs.items())))

benchmarks/measurements/2026-10-13-source-module-label-diff/compare_labels.py

@@ -1,23 +1,45 @@
 """Compare the baseline labels (old modules) with the labels built on source.py, sentinel-normalised."""
-import glob, json, collections, sys
+
+import collections
+import glob
+import json
+import sys
+
+
 def load(pattern):
     files = glob.glob(pattern)
     if not files:
         sys.exit(f"no labels file for {pattern}")
-    rows = [json.loads(l) for l in open(files[0]) if l.strip()]
+    with open(files[0]) as stream:
+        rows = [json.loads(line) for line in stream if line.strip()]
     return files[0], rows
+
+
 bfile, before = load("/home/mc/ousast-results/fossil/p1-baseline/out/labels-*.jsonl")
 afile, after = load("/home/mc/ousast-results/fossil/p1-after/out/labels-*.jsonl")
+
+
 def key(r):
     s = json.dumps(r, sort_keys=True).replace("<global>", "<module>")
     return s
-B = collections.Counter(key(r) for r in before); A = collections.Counter(key(r) for r in after)
-only_b = B - A; only_a = A - B
-print("baseline:", bfile, len(before), "rows"); print("after:   ", afile, len(after), "rows")
+
+
+B = collections.Counter(key(r) for r in before)
+A = collections.Counter(key(r) for r in after)
+only_b = B - A
+only_a = A - B
+print("baseline:", bfile, len(before), "rows")
+print("after:   ", afile, len(after), "rows")
 print("identical after sentinel normalisation:", len(only_b) == 0 and len(only_a) == 0)
 print("rows only in baseline:", sum(only_b.values()), " rows only in after:", sum(only_a.values()))
+
+
 def summarize(c, title):
     print(f"--- {title} (first 8)")
-    for s, n in list(c.items())[:8]:
-        r = json.loads(s); print("  ", {k: r.get(k) for k in ("candidate", "family", "label", "source", "unit", "pin_role") if k in r})
-summarize(only_b, "only baseline"); summarize(only_a, "only after")
+    for s in list(c)[:8]:
+        r = json.loads(s)
+        print("  ", {k: r.get(k) for k in ("candidate", "family", "label", "source", "unit", "pin_role") if k in r})
+
+
+summarize(only_b, "only baseline")
+summarize(only_a, "only after")

benchmarks/measurements/2026-10-13-span-engines/span_eval.py

@@ -38,8 +38,19 @@ from openultrasast.source import (  # noqa: E402
 
 TIMEOUT_S = 10
 EXT_LANG = {
-    ".py": "python", ".js": "javascript", ".mjs": "javascript", ".ts": "typescript", ".tsx": "typescript",
-    ".java": "java", ".c": "cpp", ".cc": "cpp", ".cpp": "cpp", ".in": "perl", ".go": "go", ".rb": "ruby", ".php": "php",
+    ".py": "python",
+    ".js": "javascript",
+    ".mjs": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".java": "java",
+    ".c": "cpp",
+    ".cc": "cpp",
+    ".cpp": "cpp",
+    ".in": "perl",
+    ".go": "go",
+    ".rb": "ruby",
+    ".php": "php",
 }
 LIZARD_EXT = {".in": ".pl"}  # lizard picks its reader by extension; the *.in pairs are Perl (c_rehash.in), which lizard has no reader for
 ANON = {"<anon>", "(anonymous)", "", "<module>"}
@@ -121,8 +132,18 @@ def lizard_spans(info) -> dict[str, tuple[int, int]]:
     return out
 
 
-TS_LANG = {"perl": None, "python": "python", "javascript": "javascript", "typescript": "typescript", "java": "java", "cpp": "cpp",
-           "c": "c", "go": "go", "ruby": "ruby", "php": "php"}
+TS_LANG = {
+    "perl": None,
+    "python": "python",
+    "javascript": "javascript",
+    "typescript": "typescript",
+    "java": "java",
+    "cpp": "cpp",
+    "c": "c",
+    "go": "go",
+    "ruby": "ruby",
+    "php": "php",
+}
 TS_FUNC = {
     "python": {"function_definition"},
     "javascript": {"function_declaration", "method_definition", "arrow_function", "function_expression", "generator_function_declaration"},
@@ -222,7 +243,8 @@ def corpus() -> tuple[list[tuple[Path, str]], list[tuple[Path, str, str]]]:
                 labels.append((c.vuln_file.resolve(), row.function, c.name))
                 labels.append((c.fixed_file.resolve(), c.fix_function or row.function, c.name))
     files: list[tuple[Path, str]] = []
-    candidates = set(lang_of) | {f.resolve() for f in (REPO / "benchmarks/pairs").rglob("*") if re.fullmatch(r"(vuln|fixed)\.[a-z]+", f.name)}
+    pair_files = (REPO / "benchmarks/pairs").rglob("*")
+    candidates = set(lang_of) | {f.resolve() for f in pair_files if re.fullmatch(r"(vuln|fixed)\.[a-z]+", f.name)}
     for f in sorted(candidates):
         if not f.is_file() or "/private/" in f.as_posix():
             continue
@@ -231,7 +253,7 @@ def corpus() -> tuple[list[tuple[Path, str]], list[tuple[Path, str, str]]]:
             continue
         files.append((f.resolve(), lang))
     seen = set()
-    labels = [l for l in labels if not (l[:2] in seen or seen.add(l[:2]))]
+    labels = [label for label in labels if not (label[:2] in seen or seen.add(label[:2]))]
     return files, labels
 
 
@@ -257,7 +279,7 @@ def spearman(xs, ys) -> float | None:
         return None
     rx, ry = _ranks(xs), _ranks(ys)
     mx, my = statistics.mean(rx), statistics.mean(ry)
-    num = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
+    num = sum((a - mx) * (b - my) for a, b in zip(rx, ry, strict=True))
     den = (sum((a - mx) ** 2 for a in rx) * sum((b - my) ** 2 for b in ry)) ** 0.5
     return None if den == 0 else num / den
 
@@ -301,7 +323,7 @@ def main() -> None:
             covered = set()
             for f in info.function_list:
                 covered.update(range(f.start_line, f.end_line + 1))
-            code_lines = [i + 1 for i, l in enumerate(lines) if l.strip()]
+            code_lines = [i + 1 for i, line in enumerate(lines) if line.strip()]
             rec["liz"] = {
                 "max_nd": max((f.max_nesting_depth for f in info.function_list), default=0),
                 "max_ccn": max((f.cyclomatic_complexity for f in info.function_list), default=0),
@@ -401,8 +423,12 @@ def main() -> None:
         "coverage": {k: {e: v for e, v in d.items()} for k, d in cov.items()},
         "agreement": {
             f"{o}~{c}": {
-                k: {"n": len(v), "start_eq": sum(x for x, _ in v) / len(v), "end_mad": statistics.mean(y for _, y in v),
-                    "end_within_1": sum(1 for _, y in v if y <= 1) / len(v)}
+                k: {
+                    "n": len(v),
+                    "start_eq": sum(x for x, _ in v) / len(v),
+                    "end_mad": statistics.mean(y for _, y in v),
+                    "end_within_1": sum(1 for _, y in v if y <= 1) / len(v),
+                }
                 for k, v in d.items()
             }
             for (o, c), d in agree.items()
@@ -411,31 +437,32 @@ def main() -> None:
         "nesting": corr,
         "module_share": {k: {"module_code_lines": v[0], "code_lines": v[1], "files_without_units": v[2]} for k, v in module.items()},
         "candidate_misses": {e: v[:40] for e, v in misses.items()},
-        "lang_counts": dict(sorted(defaultdict(int, {l: sum(1 for _, x in files if x == l) for l in set(x for _, x in files)}).items())),
+        "lang_counts": dict(sorted({lang: sum(1 for _, x in files if x == lang) for lang in {x for _, x in files}}.items())),
     }
     (OUT / "results.json").write_text(json.dumps(out, indent=1, default=str))
 
     # markdown tables
     md = []
-    langs = [l for l in ("cpp", "python", "java", "javascript", "typescript", "perl", "all") if l in cov]
+    langs = [lang for lang in ("cpp", "python", "java", "javascript", "typescript", "perl", "all") if lang in cov]
     md.append("### Coverage of labelled functions (hit/total)\n")
     md.append("| language | " + " | ".join(engines) + " |")
     md.append("|---|" + "---|" * len(engines))
-    for l in langs:
-        md.append(f"| {l} | " + " | ".join(f"{cov[l][e][0]}/{cov[l][e][1]} ({cov[l][e][0] / max(cov[l][e][1], 1):.0%})" for e in engines) + " |")
+    for lang in langs:
+        cells = (f"{cov[lang][e][0]}/{cov[lang][e][1]} ({cov[lang][e][0] / max(cov[lang][e][1], 1):.0%})" for e in engines)
+        md.append(f"| {lang} | " + " | ".join(cells) + " |")
     md.append("\n### Span agreement on labelled functions both found (start equal %, end mean abs diff, end within 1 line %)\n")
     md.append("| pair | language | n | start= | end MAD | end<=1 |")
     md.append("|---|---|---|---|---|---|")
     for key, d in out["agreement"].items():
-        for l in langs:
-            if l in d:
-                v = d[l]
-                md.append(f"| {key} | {l} | {v['n']} | {v['start_eq']:.0%} | {v['end_mad']:.2f} | {v['end_within_1']:.0%} |")
+        for lang in langs:
+            if lang in d:
+                v = d[lang]
+                md.append(f"| {key} | {lang} | {v['n']} | {v['start_eq']:.0%} | {v['end_mad']:.2f} | {v['end_within_1']:.0%} |")
     md.append("\n### Name-set Jaccard per file (mean)\n")
     md.append("| pair | " + " | ".join(langs) + " |")
     md.append("|---|" + "---|" * len(langs))
     for key, d in out["jaccard"].items():
-        md.append(f"| {key} | " + " | ".join(f"{d[l]['mean']:.2f} (n={d[l]['n']})" if l in d else "-" for l in langs) + " |")
+        md.append(f"| {key} | " + " | ".join(f"{d[lang]['mean']:.2f} (n={d[lang]['n']})" if lang in d else "-" for lang in langs) + " |")
     md.append("\n### Runtime (seconds per 1,000 files, this corpus) and failures\n")
     md.append("| engine | s/1000 files | failures |")
     md.append("|---|---|---|")
@@ -444,16 +471,17 @@ def main() -> None:
     md.append("\n### lizard nesting/CCN vs measure_nesting (file level, Spearman)\n")
     md.append("| set | n | rho(max_nesting_depth) | rho(max CCN) | rho(sum CCN) | exact == max_nesting_depth |")
     md.append("|---|---|---|---|---|---|")
-    for l, v in corr.items():
+    for lang, v in corr.items():
         f = lambda x: "-" if x is None else f"{x:.2f}"  # noqa: E731
-        md.append(f"| {l} | {v['n']} | {f(v['spearman_max_nesting_depth'])} | {f(v['spearman_max_ccn'])} | {f(v['spearman_sum_ccn'])} | {v['exact_equal_max_nd']} |")
+        rho = f"{f(v['spearman_max_nesting_depth'])} | {f(v['spearman_max_ccn'])} | {f(v['spearman_sum_ccn'])}"
+        md.append(f"| {lang} | {v['n']} | {rho} | {v['exact_equal_max_nd']} |")
     md.append("\n### Code lines outside any lizard unit (module-level share)\n")
     md.append("| language | module code lines | code lines | share | files with zero units |")
     md.append("|---|---|---|---|---|")
-    for l in langs:
-        if l in module:
-            v = module[l]
-            md.append(f"| {l} | {v[0]} | {v[1]} | {v[0] / max(v[1], 1):.0%} | {v[2]} |")
+    for lang in langs:
+        if lang in module:
+            v = module[lang]
+            md.append(f"| {lang} | {v[0]} | {v[1]} | {v[0] / max(v[1], 1):.0%} | {v[2]} |")
     (OUT / "tables.md").write_text("\n".join(md) + "\n")
     print("\n".join(md))
     print("\nfailures:", json.dumps(failures, indent=1))