mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(hunt): diff producer records where a push removed lines; worktree checkout is proven complete
Two instrument findings from the first access-control feature table (952 rows, 74 positive rows): 1. A fix that adds a guard is, replayed base-to-head, a push that removes it. The diff producer only kept hunks with added lines at head, so 38 of 74 positive rows had no hunk in their file at all and 69 of 74 were "not on a changed line", while 60% of ordinary rows were. The diff annotation separated the labels by construction and was blind to exactly the access-control shape. The producer (version 2) now records per file the head lines after which the push removed code; distance and on_changed_line count them, the model's hunks input renders them as removed_after_line, and diff.deletion_count is an engineered feature. 2. 150 rows (16%, every positive of one repository) read empty files from the worktree although git holds the blobs; the build ran while another process checked out from the same clone. _full_checkout now proves the checkout: git status must be clean, a dirty tree is restored once, then InstrumentFailure checkout_incomplete. The table script cross-checks every unread row against the blob size and refuses to screen a partly unread tree, and records the diff-scoped recall ceiling (positive rows a label-blind enumeration would reach). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
60cbefec9b
commit
537b61e8d9
@@ -0,0 +1,149 @@
|
||||
"""Feature table and separation screen over a frozen training pool, no model calls.
|
||||
|
||||
For every change of the pool's slice, check out its head, enumerate the candidate rows (touched functions at head
|
||||
plus the labelled fix site) and compute the four producers. Rows are labelled 1 at an accepted fix site, 0 on an
|
||||
ordinary push and null elsewhere on a positive. The screen then scores every engineered feature alone against the
|
||||
label (ROC-AUC) and reports the strongest feature per annotation. Repositories are the clusters: with few of them
|
||||
the screen is a direction, not a result, and the output says how many there are.
|
||||
|
||||
PYTHONPATH=. .venv/bin/python benchmarks/measurements/2026-10-13-access-control-screen/table.py \\
|
||||
benchmarks/valid/train/pool-train.toml --repos ~/ousast-results/valid/train/repos_hist.json --out <dir>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import tomllib
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.hunt import features
|
||||
from openultrasast.hunt.producers import compute_rows
|
||||
from openultrasast.hunt.runner import _full_checkout as full_checkout
|
||||
from openultrasast.hunt.table import FeatureTable
|
||||
from openultrasast.stats import separation
|
||||
|
||||
|
||||
def _git(root: Path, *args: str) -> str:
|
||||
return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def changes_of(pool: Path, slice_id: int) -> list[dict]:
|
||||
data = tomllib.loads(pool.read_text())
|
||||
rows = next(value for value in data.values() if isinstance(value, list) and value and "changes" in value[0])
|
||||
out = []
|
||||
for row in rows:
|
||||
if int(row.get("slice", 1)) != slice_id:
|
||||
continue
|
||||
for change in row["changes"]:
|
||||
change = dict(change)
|
||||
change.setdefault("repo", row.get("repository") or row.get("repo") or row["id"])
|
||||
change.setdefault("id", change["case_id"])
|
||||
out.append(change)
|
||||
return out
|
||||
|
||||
|
||||
def rows_for(change: dict, clone: Path) -> tuple[list[dict], dict]:
|
||||
started = time.monotonic()
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-table-") as temporary:
|
||||
root = Path(temporary) / "head"
|
||||
_git(clone, "worktree", "add", "--detach", str(root), change["head"])
|
||||
try:
|
||||
if _git(root, "rev-parse", "HEAD").strip() != change["head"]:
|
||||
raise RuntimeError("head_mismatch")
|
||||
full_checkout(root)
|
||||
rows = compute_rows(root, change)
|
||||
finally:
|
||||
_git(clone, "worktree", "remove", "--force", str(root))
|
||||
return rows, {"seconds": round(time.monotonic() - started, 2), "rows": len(rows)}
|
||||
|
||||
|
||||
def unread_rows(rows: list[dict], repos: dict[str, str]) -> int:
|
||||
"""Rows whose file the source producer could not read although git holds a non-empty blob at head.
|
||||
|
||||
An empty or missing file in the worktree is a checkout failure, not a feature; the first build read 150 such
|
||||
rows as feature-free (every positive of one repository among them) and nothing in the screen said so."""
|
||||
count = 0
|
||||
checked: dict[tuple[str, str, str], bool] = {}
|
||||
for row in rows:
|
||||
if row.get("provenance.source.reason") not in ("empty_source", "FileNotFoundError"):
|
||||
continue
|
||||
key = (row["repo"], row["head"], row["path"])
|
||||
if key not in checked:
|
||||
clone = Path(repos[row["repo"]]).expanduser()
|
||||
size = subprocess.run(
|
||||
["git", "-C", str(clone), "cat-file", "-s", f"{row['head']}:{row['path']}"], capture_output=True, text=True
|
||||
)
|
||||
checked[key] = size.returncode == 0 and int(size.stdout.strip() or 0) > 0
|
||||
count += checked[key]
|
||||
return count
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("pool", type=Path)
|
||||
parser.add_argument("--repos", type=Path, required=True, help="JSON map repo -> local clone with history")
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--slice", type=int, default=1)
|
||||
args = parser.parse_args(argv)
|
||||
repos = json.loads(args.repos.read_text())
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
rows, log = [], []
|
||||
for change in changes_of(args.pool, args.slice):
|
||||
clone = Path(repos[change["repo"]]).expanduser()
|
||||
try:
|
||||
produced, info = rows_for(change, clone)
|
||||
except Exception as error: # noqa: BLE001 (type only; a message can carry a path)
|
||||
info = {"failure": type(error).__name__}
|
||||
produced = []
|
||||
rows.extend(produced)
|
||||
log.append({"id": change["id"], "repo": change["repo"], "kind": change["kind"], **info})
|
||||
print(f"{change['id'][:16]} {change['kind']} {info}", flush=True)
|
||||
table = FeatureTable(rows)
|
||||
table.write_jsonl(args.out / "table.jsonl")
|
||||
# The instrument gate: a producer that failed on every row saw an unread tree, not a feature-free one.
|
||||
provenance = {name: sum(bool(row.get(f"provenance.{name}.ok")) for row in rows) for name in ("diff", "source", "semantic", "taxonomy")}
|
||||
print("producer rows ok:", provenance, "of", len(rows), flush=True)
|
||||
if rows and provenance["source"] == 0:
|
||||
print("the source producer read nothing: refusing to screen an unread tree", file=sys.stderr)
|
||||
return 2
|
||||
unread = unread_rows(rows, repos)
|
||||
print("rows unread although git holds the file:", unread, flush=True)
|
||||
if unread:
|
||||
print("the checkout was incomplete for some rows: refusing to screen a partly unread tree", file=sys.stderr)
|
||||
return 2
|
||||
(args.out / "log.json").write_text(json.dumps(log, indent=1))
|
||||
frame = features.feature_frame(table)
|
||||
labelled = frame[frame["label"].isin([0, 1])]
|
||||
summary = {
|
||||
"producer_rows_ok": provenance,
|
||||
"table_digest": table.digest(),
|
||||
"rows": len(frame),
|
||||
"labelled": {"positive": int((labelled["label"] == 1).sum()), "negative": int((labelled["label"] == 0).sum())},
|
||||
"repositories": int(labelled["repo"].nunique()),
|
||||
"changes": int(labelled["change_id"].nunique()),
|
||||
"positive_repositories": int(labelled[labelled["label"] == 1]["repo"].nunique()),
|
||||
"source_failures": dict(Counter(row.get("provenance.source.reason") for row in rows if not row.get("provenance.source.ok"))),
|
||||
"unread_rows": unread,
|
||||
# The recall ceiling of a diff-scoped candidate set: positive rows a label-blind enumeration would reach.
|
||||
"positive_rows_on_changed_line": int(labelled[(labelled["label"] == 1) & (labelled["diff.on_changed_line"] == 1)].shape[0]),
|
||||
"positive_rows_with_deletions": int(labelled[(labelled["label"] == 1) & (labelled["diff.deletion_count"].fillna(0) > 0)].shape[0]),
|
||||
}
|
||||
columns = [name for name in features.ENGINEERED if name in frame.columns]
|
||||
per_feature = separation(labelled, "label", columns)
|
||||
per_feature["annotation"] = [features.ENGINEERED[name][1] for name in per_feature["column"]]
|
||||
per_feature.to_csv(args.out / "separation.csv", index=False)
|
||||
features.screen(labelled).to_csv(args.out / "screen.csv", index=False)
|
||||
(args.out / "summary.json").write_text(json.dumps(summary, indent=1))
|
||||
print(json.dumps(summary))
|
||||
print(per_feature.sort_values("auc", key=lambda s: (s - 0.5).abs(), ascending=False).to_string(index=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user