mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Merge branch 'graph-slice'
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"arms": [
|
||||
"B"
|
||||
],
|
||||
"client_calls": 2394,
|
||||
"experiment": "exp-005-graph-slice",
|
||||
"families": {
|
||||
"access_control": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 330,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 0.9956892000000007,
|
||||
"filled": 330,
|
||||
"missing": 330,
|
||||
"requests": 330,
|
||||
"still_missing": 0,
|
||||
"units": 66
|
||||
},
|
||||
"deserialization": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 310,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 0.8843295999999988,
|
||||
"filled": 310,
|
||||
"missing": 310,
|
||||
"requests": 310,
|
||||
"still_missing": 0,
|
||||
"units": 64
|
||||
},
|
||||
"injection": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 675,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 2.270278999999999,
|
||||
"filled": 675,
|
||||
"missing": 675,
|
||||
"requests": 675,
|
||||
"still_missing": 0,
|
||||
"units": 149
|
||||
},
|
||||
"output_encoding": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 325,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 1.0014180000000004,
|
||||
"filled": 328,
|
||||
"missing": 325,
|
||||
"requests": 328,
|
||||
"still_missing": 0,
|
||||
"units": 69
|
||||
},
|
||||
"path": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 320,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 0.9399192000000005,
|
||||
"filled": 325,
|
||||
"missing": 320,
|
||||
"requests": 325,
|
||||
"still_missing": 0,
|
||||
"units": 66
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 425,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 1.174316,
|
||||
"filled": 426,
|
||||
"missing": 425,
|
||||
"requests": 426,
|
||||
"still_missing": 0,
|
||||
"units": 87
|
||||
}
|
||||
},
|
||||
"reason": "",
|
||||
"status": "done",
|
||||
"usd": 1.5890382240000016
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
{
|
||||
"alpha": 0.05,
|
||||
"arms": {
|
||||
"A": {
|
||||
"budget_usd": 0.0,
|
||||
"label": "six slice-2 incumbents unchanged, replaying exp-004's response cache",
|
||||
"repeats": 1,
|
||||
"replay_only": true
|
||||
},
|
||||
"B": {
|
||||
"budget_usd": 6.0,
|
||||
"label": "the same six programs with the in-function slice added to their prompts",
|
||||
"repeats": 1,
|
||||
"replay_only": false
|
||||
}
|
||||
},
|
||||
"code_commit": "0acbc7ec41dcaeca805c4720bc3f6d02cb940ee7",
|
||||
"decision": "inconclusive",
|
||||
"experiment": "exp-005-graph-slice",
|
||||
"hypothesis": "Adding the in-function def-use slice to the prompt improves within-pair ranking of the six slice-2 incumbent programs on held-out repositories, at no more than 1.5 times the cost per candidate.\n",
|
||||
"looks": [
|
||||
{
|
||||
"boundary": 1.959964,
|
||||
"crossed": false,
|
||||
"estimate": {
|
||||
"a": 0.7615384615384615,
|
||||
"b": 0.7461538461538462,
|
||||
"ci": [
|
||||
-0.0628019323671497,
|
||||
0.03155339805825241
|
||||
],
|
||||
"diff": -0.01538461538461533,
|
||||
"groups": 220,
|
||||
"metric": "within_pair_auc",
|
||||
"resamples": 10000,
|
||||
"se": 0.024030780235707972,
|
||||
"units": 501
|
||||
},
|
||||
"fraction": 1.0,
|
||||
"groups": 220,
|
||||
"z": -0.6402045723740141
|
||||
}
|
||||
],
|
||||
"manifest_digest": "1b1ec814cde12801491376d2086c8c2d6db70f4a86fc308c0adc5a530398e554",
|
||||
"mde": 0.0,
|
||||
"primary": [
|
||||
{
|
||||
"name": "within_pair_auc"
|
||||
}
|
||||
],
|
||||
"reason": "the CI of within_pair_auc spans 0; the incumbent stays",
|
||||
"resamples": 10000,
|
||||
"secondary": [
|
||||
{
|
||||
"name": "top_30_precision"
|
||||
},
|
||||
{
|
||||
"name": "unsure_rate"
|
||||
},
|
||||
{
|
||||
"name": "usd_per_candidate"
|
||||
}
|
||||
],
|
||||
"spend_usd": {
|
||||
"A": 1.444503,
|
||||
"B": 1.589038
|
||||
},
|
||||
"tests": {
|
||||
"top_30_precision": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 1.0,
|
||||
"b": 1.0,
|
||||
"ci": [
|
||||
0.0,
|
||||
0.0
|
||||
],
|
||||
"diff": 0.0,
|
||||
"groups": 220,
|
||||
"metric": "top_30_precision",
|
||||
"resamples": 10000,
|
||||
"se": 0.0006666666666666664,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"unsure_rate": {
|
||||
"ci_vs_zero": "favour",
|
||||
"direction": -1,
|
||||
"estimate": {
|
||||
"a": 0.29141716566866266,
|
||||
"b": 0.23952095808383234,
|
||||
"ci": [
|
||||
-0.08583690987124465,
|
||||
-0.01996007984031939
|
||||
],
|
||||
"diff": -0.05189620758483032,
|
||||
"groups": 220,
|
||||
"metric": "unsure_rate",
|
||||
"resamples": 10000,
|
||||
"se": 0.01685991406098136,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": {
|
||||
"b": 47,
|
||||
"c": 21,
|
||||
"concordant": 433,
|
||||
"p_value": 0.0021862431776861705
|
||||
},
|
||||
"target": null
|
||||
},
|
||||
"usd_per_candidate": {
|
||||
"ci_vs_zero": "against",
|
||||
"direction": -1,
|
||||
"estimate": {
|
||||
"a": 0.002883239520958083,
|
||||
"b": 0.0031717325349301397,
|
||||
"ci": [
|
||||
0.0002458205128205199,
|
||||
0.0003319800399201574
|
||||
],
|
||||
"diff": 0.0002884930139720565,
|
||||
"groups": 220,
|
||||
"metric": "usd_per_candidate",
|
||||
"resamples": 10000,
|
||||
"se": 2.1746161270931252e-05,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"within_pair_auc": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.7615384615384615,
|
||||
"b": 0.7461538461538462,
|
||||
"ci": [
|
||||
-0.0628019323671497,
|
||||
0.03155339805825241
|
||||
],
|
||||
"diff": -0.01538461538461533,
|
||||
"groups": 220,
|
||||
"metric": "within_pair_auc",
|
||||
"resamples": 10000,
|
||||
"se": 0.024030780235707972,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
}
|
||||
},
|
||||
"tost": null,
|
||||
"units": {
|
||||
"by_label": {
|
||||
"0": 218,
|
||||
"1": 283
|
||||
},
|
||||
"groups": 220,
|
||||
"outcomes": 1002,
|
||||
"paired": 501
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Read-only, zero-model slice audit. Run: env -u OPENROUTER_API_KEY python benchmarks/learn/slice_leak_audit.py.
|
||||
|
||||
Only sources.toml itself is read, never its referenced corpora. Stored source IDs
|
||||
are the reporting strata (the store merges advisory pair catalogs into `pairs`).
|
||||
Ties get half credit. Both signed and best-direction scores are recorded; like
|
||||
learn.leaks, a reversed label shortcut is still a leak. Missing classes/pairs
|
||||
produce null metrics, not plausible zeros. Unsupported slices score zero for both
|
||||
indicators and remain separately counted in coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openultrasast.config import load_dotenv
|
||||
from openultrasast.learn.examples import load_examples
|
||||
from openultrasast.learn.labels import load_sources
|
||||
from openultrasast.learn.slice import function_slice
|
||||
from openultrasast.plane.memory import MemoryStore, open_store
|
||||
|
||||
UNITS = Path("plane/experiments/exp-004-pooled-block-gate.units.jsonl")
|
||||
OUTPUT = Path("benchmarks/measurements/2026-10-03-graph-slice-leak-audit/record.json")
|
||||
THRESHOLD = 0.70
|
||||
|
||||
|
||||
def complete_pairs(rows: Sequence[Mapping[str, Any]]) -> list[tuple[Mapping[str, Any], Mapping[str, Any]]]:
|
||||
groups: dict[tuple[str, str], list[Mapping[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
if row.get("pair"):
|
||||
groups[(row["family"], row["pair"])].append(row)
|
||||
pairs = []
|
||||
for members in groups.values():
|
||||
if len(members) == 2 and {m["label"] for m in members} == {0, 1}:
|
||||
v, f = sorted(members, key=lambda m: -m["label"])
|
||||
if v["group"] != f["group"] or v["fold"] != f["fold"]:
|
||||
raise ValueError("Pair crosses group or fold")
|
||||
pairs.append((v, f))
|
||||
elif len(members) != 1:
|
||||
raise ValueError("Ambiguous pair")
|
||||
return pairs
|
||||
|
||||
|
||||
def metrics(rows: Sequence[Mapping[str, Any]], signal: str) -> dict[str, Any]:
|
||||
pairs = complete_pairs(rows)
|
||||
higher = sum(v[signal] > f[signal] for v, f in pairs)
|
||||
lower = sum(v[signal] < f[signal] for v, f in pairs)
|
||||
ties = len(pairs) - higher - lower
|
||||
pair_score = (higher + 0.5 * ties) / len(pairs) if pairs else None
|
||||
positive = [r[signal] for r in rows if r["label"] == 1]
|
||||
negative = [r[signal] for r in rows if r["label"] == 0]
|
||||
auc = (
|
||||
sum((v > f) + 0.5 * (v == f) for v in positive for f in negative) / (len(positive) * len(negative))
|
||||
if positive and negative
|
||||
else None
|
||||
)
|
||||
best_pair = max(pair_score, 1 - pair_score) if pair_score is not None else None
|
||||
best_auc = max(auc, 1 - auc) if auc is not None else None
|
||||
return {
|
||||
"rows": len(rows),
|
||||
"pairs": len(pairs),
|
||||
"positive": len(positive),
|
||||
"negative": len(negative),
|
||||
"higher_on_vulnerable": higher,
|
||||
"higher_on_fixed": lower,
|
||||
"ties": ties,
|
||||
"within_pair_accuracy": pair_score,
|
||||
"pooled_auc": auc,
|
||||
"best_direction_within_pair_accuracy": best_pair,
|
||||
"best_direction_pooled_auc": best_auc,
|
||||
"flagged": any(v is not None and v >= THRESHOLD for v in (best_pair, best_auc)),
|
||||
}
|
||||
|
||||
|
||||
def audit(store: MemoryStore, units: Sequence[Mapping[str, Any]], *, expected_rows: int = 501, expected_pairs: int = 195) -> dict[str, Any]:
|
||||
if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
|
||||
raise ValueError(f"Expected {expected_rows} unique units, read {len(units)}")
|
||||
if len(complete_pairs(units)) != expected_pairs:
|
||||
raise ValueError(f"Expected {expected_pairs} complete pairs")
|
||||
examples = load_examples(store)
|
||||
index = {e.id: e for e in examples}
|
||||
if len(index) != len(examples):
|
||||
raise ValueError("Duplicate example IDs in memory")
|
||||
sources = {s.id for s in load_sources().sources}
|
||||
coverage: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
rows = []
|
||||
excerpt_bytes = 0
|
||||
blobs: dict[str, str] = {}
|
||||
for unit in units:
|
||||
e = index[unit["unit"]] # missing input is fatal, never counted as no-flow
|
||||
if (e.family, e.label, e.group) != (unit["family"], unit["label"], unit["group"]):
|
||||
raise ValueError(f"Unit metadata differs from store: {e.id}")
|
||||
if e.source not in sources:
|
||||
raise ValueError(f"Source absent from sources.toml: {e.source}")
|
||||
if e.excerpt_sha not in blobs:
|
||||
# MemoryStore.get_blob(verify=True) DELETES corrupt blobs: do not use it.
|
||||
data = store.get_blob("excerpts", e.excerpt_sha, verify=False)
|
||||
if not data or hashlib.sha256(data).hexdigest() != e.excerpt_sha:
|
||||
raise ValueError(f"Missing, empty or corrupt excerpt: {e.id}")
|
||||
excerpt_bytes += len(data)
|
||||
blobs[e.excerpt_sha] = data.decode("utf-8")
|
||||
result = function_slice(blobs[e.excerpt_sha], e.language, e.family)
|
||||
status = "not-analysable" if result is None else "flow" if result.steps else "no-flow"
|
||||
coverage[e.family][status] += 1
|
||||
rows.append(
|
||||
{
|
||||
**unit,
|
||||
"source": e.source,
|
||||
"status": status,
|
||||
"has_flow": int(status == "flow"),
|
||||
"slice_length": len(result.steps) if result else 0,
|
||||
}
|
||||
)
|
||||
pairs = complete_pairs(rows)
|
||||
asymmetry = Counter(
|
||||
"both" if v["has_flow"] and f["has_flow"] else "vulnerable_only" if v["has_flow"] else "fixed_only" if f["has_flow"] else "neither"
|
||||
for v, f in pairs
|
||||
)
|
||||
scopes = {"all": rows, **{f"source:{s}": [r for r in rows if r["source"] == s] for s in sorted({r["source"] for r in rows})}}
|
||||
return {
|
||||
"cost_usd": 0,
|
||||
"threshold": THRESHOLD,
|
||||
"unit_rows_read": len(units),
|
||||
"example_rows_read": len(examples),
|
||||
"matched_rows": len(rows),
|
||||
"excerpts_read": len(blobs),
|
||||
"excerpt_units": len(rows),
|
||||
"excerpt_bytes": excerpt_bytes,
|
||||
"complete_pairs": len(pairs),
|
||||
"coverage": {f: {s: c[s] for s in ("flow", "no-flow", "not-analysable")} for f, c in sorted(coverage.items())},
|
||||
"pair_asymmetry": {k: asymmetry[k] for k in ("vulnerable_only", "fixed_only", "both", "neither")},
|
||||
"signals": {
|
||||
signal: {scope: metrics(members, signal) for scope, members in scopes.items()} for signal in ("has_flow", "slice_length")
|
||||
},
|
||||
"source_strata": "Stored source IDs from sources.toml; advisory pair catalogs are merged into pairs by the example builder.",
|
||||
"limitations": (
|
||||
"Flat IR: no dominance/guards, textual branch order, potential parameter inputs, conservative call returns; no alias analysis."
|
||||
),
|
||||
"units": rows,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv(Path(".env"))
|
||||
os.environ.pop("OPENROUTER_API_KEY", None) # dotenv must not restore the removed key
|
||||
raw = UNITS.read_bytes()
|
||||
units = [json.loads(line) for line in raw.decode().splitlines() if line.strip()]
|
||||
report = audit(open_store(), units)
|
||||
report["units_bytes"] = len(raw)
|
||||
report["units_sha256"] = hashlib.sha256(raw).hexdigest()
|
||||
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUTPUT.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
totals = Counter({s: sum(c[s] for c in report["coverage"].values()) for s in ("flow", "no-flow", "not-analysable")})
|
||||
lines = [
|
||||
f"Slice leak audit: $0; units bytes={len(raw)}",
|
||||
f"Rows read: units={len(units)}, examples={report['example_rows_read']}, matched={report['matched_rows']}",
|
||||
f"Excerpts read: {report['excerpts_read']}; units={report['excerpt_units']}; bytes={report['excerpt_bytes']}",
|
||||
f"Coverage: {dict(totals)}",
|
||||
f"Complete pairs: {report['complete_pairs']}",
|
||||
f"Pair asymmetry: {report['pair_asymmetry']}",
|
||||
f"Has flow: {report['signals']['has_flow']['all']}",
|
||||
f"Slice length: {report['signals']['slice_length']['all']}",
|
||||
f"Source strata: {', '.join(report['signals']['has_flow'])}; threshold={THRESHOLD}",
|
||||
f"Report: {OUTPUT}",
|
||||
]
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user