"""Offline audit of completed Joern results; never launches an analyzer or reads corpora. Missing results stay in coverage, not signal denominators. Observed non-path rows score zero for path features. Sink kinds are one-hot indicators, never ordinal category IDs. Both signed and best-direction metrics expose reversed shortcuts. """ from __future__ import annotations import argparse import ast import hashlib import json import re from collections import Counter from pathlib import Path from slice_leak_audit import THRESHOLD, complete_pairs, metrics ROOT = Path(__file__).resolve().parents[2] UNITS = ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl" RESULTS = Path.home() / "ousast-results/plane/engine-trace" OUTPUT = ROOT / "benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json" STATUSES = ("path", "asked-nothing", "failed", "timeout", "unsupported", "missing") ASKED = {"path", "asked-nothing"} SIGNALS = ("has_path", "shortest_path_steps", "sanitized", "guard_present", "asked") ASYMMETRY = ("vulnerable_only", "fixed_only", "both", "neither", "not_both_asked") def materialization_version(): # Read the producer's literal without importing its execution dependencies. tree = ast.parse(Path(__file__).with_name("engine_trace.py").read_bytes()) for node in tree.body: if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "MATERIALIZATION_VERSION" for t in node.targets): return ast.literal_eval(node.value) raise ValueError("Producer has no materialization version") def features(row): path = row["status"] == "path" traces = row.get("traces", []) if path else [] candidates = [(i, t) for i, t in enumerate(traces) if t.get("steps")] index, trace = min(candidates, key=lambda it: len(it[1]["steps"])) if candidates else (None, {}) witnesses = row.get("witness_rows", []) witness = witnesses[index] if index is not None and index < len(witnesses) else {} guard = trace.get("guard", {}) sink = witness.get("sinkKind") if not sink: # Retain only the call target: arguments, literals and code are not kinds. match = re.search(r"([\w:$>.\\-]+)\s*\(", str(witness.get("sink", ""))) sink = match[1] if match else "unknown" return { "has_path": int(path), "shortest_path_steps": len(trace["steps"]) if trace else None if path else 0, "sanitized": int(bool(trace.get("sanitized", witness.get("sanitized", False)))) if trace else None if path else 0, "guard_present": int( bool(guard.get("text") or guard.get("bounded") or witness.get("bound") or witness.get("bounded")) or any("guard" in s.get("roles", []) for s in trace.get("steps", [])) ) if trace else None if path else 0, "sink_kind": str(sink) if path else "no-path", "asked": int(row["status"] in ASKED), } def scored(rows, signal): eligible = [r for r in rows if r[signal] is not None] result = metrics(eligible, signal) result["excluded_rows"] = len(rows) - len(eligible) result["flagged"] = (result["pairs"] >= 10 and (result["best_direction_within_pair_accuracy"] or 0) >= THRESHOLD) or ( result["rows"] >= 10 and (result["best_direction_pooled_auc"] or 0) >= THRESHOLD ) return result def signal_metrics(rows, kinds): return { **{s: scored(rows, s) for s in SIGNALS}, "sink_kind": {kind: scored([{**r, "is_kind": int(r["sink_kind"] == kind)} for r in rows], "is_kind") for kind in kinds}, } def scope_report(rows, kinds): pairs = complete_pairs(rows) asymmetry = Counter() honest = [] for v, f in pairs: if not (v["asked"] and f["asked"]): asymmetry["not_both_asked"] += 1 else: honest.extend((v, f)) asymmetry[ "both" if v["has_path"] and f["has_path"] else "vulnerable_only" if v["has_path"] else "fixed_only" if f["has_path"] else "neither" ] += 1 observed = [r for r in rows if r["status"] != "missing"] return { "rows": len(rows), "coverage": {s: sum(r["status"] == s for r in rows) for s in STATUSES}, "complete_pairs": len(pairs), "pair_asymmetry": {s: asymmetry[s] for s in ASYMMETRY}, "both_asked_pairs": len(honest) // 2, "signals": signal_metrics(observed, kinds), "both_asked_signals": signal_metrics(honest, kinds), } def audit(units, results, *, expected_rows=501, expected_pairs=195): if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows: raise ValueError(f"Expected {expected_rows} unique units") if len(complete_pairs(units)) != expected_pairs: raise ValueError(f"Expected {expected_pairs} complete pairs") if not results.is_dir(): raise ValueError(f"Results directory is unreadable: {results}") version = materialization_version() unit_index = {u["unit"]: u for u in units} matched = {} files = [] counts = Counter() for path in sorted(results.glob("*.json")): raw = path.read_bytes() record = json.loads(raw) # Corruption fails loudly; producer uses atomic replace. if not isinstance(record, dict) or "repo" not in record or "pin" not in record: continue stale = record.get("materialization_version") != version unfinished = record.get("done") is not True counts["stale"] += stale counts["unfinished"] += unfinished counts["skipped"] += stale or unfinished files.append((path.name, len(raw))) if stale or unfinished: continue counts["accepted_files"] += 1 counts["result_rows_read"] += len(record["units"]) for row in record["units"]: if row["unit"] not in unit_index: continue unit = unit_index[row["unit"]] if any(row.get(k) != unit.get(k) for k in ("family", "label", "pair")): raise ValueError(f"Result metadata mismatch: {row['unit']}") if row["status"] not in STATUSES[:-1] or not row.get("source"): raise ValueError(f"Invalid result status/source: {row['unit']}") if row["unit"] in matched: raise ValueError(f"Duplicate current result: {row['unit']}") matched[row["unit"]] = { **unit, "source": row["source"], "status": row["status"], "pin_role": row.get("pin_role"), **features(row), } rows = [ matched.get(u["unit"], {**u, "source": u.get("source", "unknown"), "status": "missing", **features({"status": "missing"})}) for u in units ] kinds = sorted({r["sink_kind"] for r in rows if r["status"] != "missing"}) scopes = {"overall": scope_report(rows, kinds)} for field in ("family", "source"): scopes[f"by_{field}"] = { value: scope_report([r for r in rows if r[field] == value], kinds) for value in sorted({r[field] for r in rows}) } return { "cost_usd": 0, "materialization_version": version, "threshold": THRESHOLD, "minimum_samples": 10, "unit_rows_read": len(units), "matched_rows": len(matched), "inputs": { **{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")}, "files_read": len(files), "bytes_read": sum(size for _, size in files), # Hash a canonical inventory; filenames and paths may identify repositories. "files_sha256": hashlib.sha256(json.dumps(sorted(files), separators=(",", ":")).encode()).hexdigest(), }, "semantics": { "signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.", "missing": "Coverage only; absent source metadata is unknown. Skipped result units are never consumed.", "non_path": "Observed non-path rows score zero; path rows lacking trace evidence have null trace features.", "asked": "path or asked-nothing=1; failed, timeout, unsupported=0; missing excluded from metrics.", "asymmetry": "Disjoint categories; not_both_asked takes precedence over path asymmetry.", "flags": "Best of both directions, >=0.70 with >=10 pairs for pair accuracy or >=10 rows for AUC.", "sink_kind": "One binary indicator per observed call target, including no-path/unknown; no ordinal encoding.", "limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.", }, **scopes, "units": [{k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", *SIGNALS, "sink_kind")} for r in rows], } def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--results", type=Path, default=RESULTS) parser.add_argument("--units", type=Path, default=UNITS) parser.add_argument("--output", type=Path, default=OUTPUT) args = parser.parse_args() raw = args.units.read_bytes() units = [json.loads(line) for line in raw.splitlines() if line.strip()] report = audit(units, args.results) report["units_input"] = {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") overall, inputs = report["overall"], report["inputs"] lines = [ f"Joern leak audit: $0; materialization_version={report['materialization_version']}", f"Units read: {len(units)}; bytes={len(raw)}; file={args.units}", f"Result files read: {inputs['files_read']}; bytes={inputs['bytes_read']}; accepted={inputs['accepted_files']}", f"Skipped: {inputs['skipped']} (stale={inputs['stale']}, unfinished={inputs['unfinished']}; counts overlap)", f"Result rows read: {inputs['result_rows_read']}; matched={report['matched_rows']}", f"Coverage: {overall['coverage']}", f"Pairs: {overall['complete_pairs']}; both asked={overall['both_asked_pairs']}", f"Asymmetry: {overall['pair_asymmetry']}", ] for signal in (*SIGNALS, "sink_kind"): def brief(value, signal=signal): if signal == "sink_kind": return {k: brief_metric(m) for k, m in value.items()} return brief_metric(value) lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}") lines.append(f"Family/source metrics and input inventory digest: {args.output}") print("\n".join(lines)) def brief_metric(metric): return {k: metric[k] for k in ("rows", "pairs", "within_pair_accuracy", "pooled_auc", "flagged")} if __name__ == "__main__": main()