mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5.5
afff080874
The Joern leak-audit record named spent population v1/v2 repositories, which the reserved-repository guard forbids outside benchmarks/independent/ (no v3 repository was involved: 0 matches). Records now carry counts and SHA-256 digests. With Codex. Co-Authored-By: Claude Opus 5.5 <[email protected]>
240 lines
11 KiB
Python
240 lines
11 KiB
Python
"""Offline audit of completed Joern results; never launches an analyzer or reads corpora.
|
|
|
|
Missing results stay in coverage, not signal denominators. Observed non-path rows
|
|
score zero for path features. Sink kinds are one-hot indicators, never ordinal
|
|
category IDs. Both signed and best-direction metrics expose reversed shortcuts.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ast
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
from slice_leak_audit import THRESHOLD, complete_pairs, metrics
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
UNITS = ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl"
|
|
RESULTS = Path.home() / "ousast-results/plane/engine-trace"
|
|
OUTPUT = ROOT / "benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json"
|
|
STATUSES = ("path", "asked-nothing", "failed", "timeout", "unsupported", "missing")
|
|
ASKED = {"path", "asked-nothing"}
|
|
SIGNALS = ("has_path", "shortest_path_steps", "sanitized", "guard_present", "asked")
|
|
ASYMMETRY = ("vulnerable_only", "fixed_only", "both", "neither", "not_both_asked")
|
|
|
|
|
|
def materialization_version():
|
|
# Read the producer's literal without importing its execution dependencies.
|
|
tree = ast.parse(Path(__file__).with_name("engine_trace.py").read_bytes())
|
|
for node in tree.body:
|
|
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "MATERIALIZATION_VERSION" for t in node.targets):
|
|
return ast.literal_eval(node.value)
|
|
raise ValueError("Producer has no materialization version")
|
|
|
|
|
|
def features(row):
|
|
path = row["status"] == "path"
|
|
traces = row.get("traces", []) if path else []
|
|
candidates = [(i, t) for i, t in enumerate(traces) if t.get("steps")]
|
|
index, trace = min(candidates, key=lambda it: len(it[1]["steps"])) if candidates else (None, {})
|
|
witnesses = row.get("witness_rows", [])
|
|
witness = witnesses[index] if index is not None and index < len(witnesses) else {}
|
|
guard = trace.get("guard", {})
|
|
sink = witness.get("sinkKind")
|
|
if not sink:
|
|
# Retain only the call target: arguments, literals and code are not kinds.
|
|
match = re.search(r"([\w:$>.\\-]+)\s*\(", str(witness.get("sink", "")))
|
|
sink = match[1] if match else "unknown"
|
|
return {
|
|
"has_path": int(path),
|
|
"shortest_path_steps": len(trace["steps"]) if trace else None if path else 0,
|
|
"sanitized": int(bool(trace.get("sanitized", witness.get("sanitized", False)))) if trace else None if path else 0,
|
|
"guard_present": int(
|
|
bool(guard.get("text") or guard.get("bounded") or witness.get("bound") or witness.get("bounded"))
|
|
or any("guard" in s.get("roles", []) for s in trace.get("steps", []))
|
|
)
|
|
if trace
|
|
else None
|
|
if path
|
|
else 0,
|
|
"sink_kind": str(sink) if path else "no-path",
|
|
"asked": int(row["status"] in ASKED),
|
|
}
|
|
|
|
|
|
def scored(rows, signal):
|
|
eligible = [r for r in rows if r[signal] is not None]
|
|
result = metrics(eligible, signal)
|
|
result["excluded_rows"] = len(rows) - len(eligible)
|
|
result["flagged"] = (result["pairs"] >= 10 and (result["best_direction_within_pair_accuracy"] or 0) >= THRESHOLD) or (
|
|
result["rows"] >= 10 and (result["best_direction_pooled_auc"] or 0) >= THRESHOLD
|
|
)
|
|
return result
|
|
|
|
|
|
def signal_metrics(rows, kinds):
|
|
return {
|
|
**{s: scored(rows, s) for s in SIGNALS},
|
|
"sink_kind": {kind: scored([{**r, "is_kind": int(r["sink_kind"] == kind)} for r in rows], "is_kind") for kind in kinds},
|
|
}
|
|
|
|
|
|
def scope_report(rows, kinds):
|
|
pairs = complete_pairs(rows)
|
|
asymmetry = Counter()
|
|
honest = []
|
|
for v, f in pairs:
|
|
if not (v["asked"] and f["asked"]):
|
|
asymmetry["not_both_asked"] += 1
|
|
else:
|
|
honest.extend((v, f))
|
|
asymmetry[
|
|
"both"
|
|
if v["has_path"] and f["has_path"]
|
|
else "vulnerable_only"
|
|
if v["has_path"]
|
|
else "fixed_only"
|
|
if f["has_path"]
|
|
else "neither"
|
|
] += 1
|
|
observed = [r for r in rows if r["status"] != "missing"]
|
|
return {
|
|
"rows": len(rows),
|
|
"coverage": {s: sum(r["status"] == s for r in rows) for s in STATUSES},
|
|
"complete_pairs": len(pairs),
|
|
"pair_asymmetry": {s: asymmetry[s] for s in ASYMMETRY},
|
|
"both_asked_pairs": len(honest) // 2,
|
|
"signals": signal_metrics(observed, kinds),
|
|
"both_asked_signals": signal_metrics(honest, kinds),
|
|
}
|
|
|
|
|
|
def audit(units, results, *, expected_rows=501, expected_pairs=195):
|
|
if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
|
|
raise ValueError(f"Expected {expected_rows} unique units")
|
|
if len(complete_pairs(units)) != expected_pairs:
|
|
raise ValueError(f"Expected {expected_pairs} complete pairs")
|
|
if not results.is_dir():
|
|
raise ValueError(f"Results directory is unreadable: {results}")
|
|
version = materialization_version()
|
|
unit_index = {u["unit"]: u for u in units}
|
|
matched = {}
|
|
files = []
|
|
counts = Counter()
|
|
for path in sorted(results.glob("*.json")):
|
|
raw = path.read_bytes()
|
|
record = json.loads(raw) # Corruption fails loudly; producer uses atomic replace.
|
|
if not isinstance(record, dict) or "repo" not in record or "pin" not in record:
|
|
continue
|
|
stale = record.get("materialization_version") != version
|
|
unfinished = record.get("done") is not True
|
|
counts["stale"] += stale
|
|
counts["unfinished"] += unfinished
|
|
counts["skipped"] += stale or unfinished
|
|
files.append((path.name, len(raw)))
|
|
if stale or unfinished:
|
|
continue
|
|
counts["accepted_files"] += 1
|
|
counts["result_rows_read"] += len(record["units"])
|
|
for row in record["units"]:
|
|
if row["unit"] not in unit_index:
|
|
continue
|
|
unit = unit_index[row["unit"]]
|
|
if any(row.get(k) != unit.get(k) for k in ("family", "label", "pair")):
|
|
raise ValueError(f"Result metadata mismatch: {row['unit']}")
|
|
if row["status"] not in STATUSES[:-1] or not row.get("source"):
|
|
raise ValueError(f"Invalid result status/source: {row['unit']}")
|
|
if row["unit"] in matched:
|
|
raise ValueError(f"Duplicate current result: {row['unit']}")
|
|
matched[row["unit"]] = {
|
|
**unit,
|
|
"source": row["source"],
|
|
"status": row["status"],
|
|
"pin_role": row.get("pin_role"),
|
|
**features(row),
|
|
}
|
|
rows = [
|
|
matched.get(u["unit"], {**u, "source": u.get("source", "unknown"), "status": "missing", **features({"status": "missing"})})
|
|
for u in units
|
|
]
|
|
kinds = sorted({r["sink_kind"] for r in rows if r["status"] != "missing"})
|
|
scopes = {"overall": scope_report(rows, kinds)}
|
|
for field in ("family", "source"):
|
|
scopes[f"by_{field}"] = {
|
|
value: scope_report([r for r in rows if r[field] == value], kinds) for value in sorted({r[field] for r in rows})
|
|
}
|
|
return {
|
|
"cost_usd": 0,
|
|
"materialization_version": version,
|
|
"threshold": THRESHOLD,
|
|
"minimum_samples": 10,
|
|
"unit_rows_read": len(units),
|
|
"matched_rows": len(matched),
|
|
"inputs": {
|
|
**{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")},
|
|
"files_read": len(files),
|
|
"bytes_read": sum(size for _, size in files),
|
|
# Hash a canonical inventory; filenames and paths may identify repositories.
|
|
"files_sha256": hashlib.sha256(json.dumps(sorted(files), separators=(",", ":")).encode()).hexdigest(),
|
|
},
|
|
"semantics": {
|
|
"signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.",
|
|
"missing": "Coverage only; absent source metadata is unknown. Skipped result units are never consumed.",
|
|
"non_path": "Observed non-path rows score zero; path rows lacking trace evidence have null trace features.",
|
|
"asked": "path or asked-nothing=1; failed, timeout, unsupported=0; missing excluded from metrics.",
|
|
"asymmetry": "Disjoint categories; not_both_asked takes precedence over path asymmetry.",
|
|
"flags": "Best of both directions, >=0.70 with >=10 pairs for pair accuracy or >=10 rows for AUC.",
|
|
"sink_kind": "One binary indicator per observed call target, including no-path/unknown; no ordinal encoding.",
|
|
"limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.",
|
|
},
|
|
**scopes,
|
|
"units": [{k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", *SIGNALS, "sink_kind")} for r in rows],
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--results", type=Path, default=RESULTS)
|
|
parser.add_argument("--units", type=Path, default=UNITS)
|
|
parser.add_argument("--output", type=Path, default=OUTPUT)
|
|
args = parser.parse_args()
|
|
raw = args.units.read_bytes()
|
|
units = [json.loads(line) for line in raw.splitlines() if line.strip()]
|
|
report = audit(units, args.results)
|
|
report["units_input"] = {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
|
|
overall, inputs = report["overall"], report["inputs"]
|
|
lines = [
|
|
f"Joern leak audit: $0; materialization_version={report['materialization_version']}",
|
|
f"Units read: {len(units)}; bytes={len(raw)}; file={args.units}",
|
|
f"Result files read: {inputs['files_read']}; bytes={inputs['bytes_read']}; accepted={inputs['accepted_files']}",
|
|
f"Skipped: {inputs['skipped']} (stale={inputs['stale']}, unfinished={inputs['unfinished']}; counts overlap)",
|
|
f"Result rows read: {inputs['result_rows_read']}; matched={report['matched_rows']}",
|
|
f"Coverage: {overall['coverage']}",
|
|
f"Pairs: {overall['complete_pairs']}; both asked={overall['both_asked_pairs']}",
|
|
f"Asymmetry: {overall['pair_asymmetry']}",
|
|
]
|
|
for signal in (*SIGNALS, "sink_kind"):
|
|
|
|
def brief(value, signal=signal):
|
|
if signal == "sink_kind":
|
|
return {k: brief_metric(m) for k, m in value.items()}
|
|
return brief_metric(value)
|
|
|
|
lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}")
|
|
lines.append(f"Family/source metrics and input inventory digest: {args.output}")
|
|
print("\n".join(lines))
|
|
|
|
|
|
def brief_metric(metric):
|
|
return {k: metric[k] for k in ("rows", "pairs", "within_pair_accuracy", "pooled_auc", "flagged")}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|