mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
feat(stats): one statistics module on scipy, statsmodels and scikit-learn; engineered features tied to the annotations
openultrasast.stats: Wilson intervals (statsmodels), exact McNemar, a repository-clustered percentile bootstrap and its paired form with one quantile convention, grouped folds (GroupKFold, leave-one-group-out), and per-column separation (ROC AUC) for the leak audit and the signal screen. Checked against the benchmark scorer's Wilson and the learn harness's McNemar. hunt/features.py turns the feature table into the engineered frame (distances, counts, flags) and keeps the map from every engineered feature to the annotation the model is shown, so an ablation mask drops both, and screen() says which annotation carries signal before a model call is spent. The [hunt] extra now carries the libraries; the core install is unchanged. The learn harness's own copies stay until that line is revived. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
d9d6335ab3
commit
c95b9241d8
@@ -0,0 +1,123 @@
|
||||
"""Feature engineering over the feature table, and the tie between engineered features and the DSPy annotations.
|
||||
|
||||
The table's raw columns (`feature.<producer>.<name>`) are what the producers measured. This module turns them
|
||||
into the numeric frame the statistics read (distances, counts, flags), and keeps the map from every engineered
|
||||
feature back to the annotation the model is shown. An ablation that masks an annotation therefore drops both the
|
||||
rendered input and the engineered features that came from the same column, and the separation screen can say
|
||||
which annotation carries signal before any model call is spent on it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from .table import FeatureTable
|
||||
|
||||
# engineered feature -> (raw column, annotation input name in program.INPUTS)
|
||||
ENGINEERED: dict[str, tuple[str, str]] = {
|
||||
"diff.distance": ("feature.diff.distance", "hunks"),
|
||||
"diff.on_changed_line": ("feature.diff.on_changed_line", "hunks"),
|
||||
"diff.hunk_count": ("feature.diff.hunks", "hunks"),
|
||||
"diff.hunk_lines": ("feature.diff.hunks", "hunks"),
|
||||
"source.span_lines": ("feature.source.span", "function_text"),
|
||||
"source.is_module": ("feature.source.function", "change_summary"),
|
||||
"source.caller_count": ("feature.source.callers", "callers"),
|
||||
"source.caller_files": ("feature.source.callers", "callers"),
|
||||
"source.entry_point_count": ("feature.source.entry_points", "entry_points"),
|
||||
"semantic.sources": ("feature.semantic.roles", "roles"),
|
||||
"semantic.sinks": ("feature.semantic.roles", "roles"),
|
||||
"semantic.sanitizers": ("feature.semantic.roles", "roles"),
|
||||
"semantic.binds": ("feature.semantic.def_use", "def_use"),
|
||||
"semantic.calls": ("feature.semantic.def_use", "def_use"),
|
||||
"semantic.obligations": ("feature.semantic.obligations", "obligations"),
|
||||
"semantic.obligations_undischarged": ("feature.semantic.obligations", "obligations"),
|
||||
}
|
||||
KEYS = ("change_id", "repo", "head", "path", "line", "kind", "fold", "label", "family")
|
||||
|
||||
|
||||
def _count(value: object) -> int | None:
|
||||
return None if value is None else len(value) if isinstance(value, (list, dict, tuple)) else None
|
||||
|
||||
|
||||
def _engineer(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
hunks = row.get("feature.diff.hunks")
|
||||
span = row.get("feature.source.span")
|
||||
callers = row.get("feature.source.callers")
|
||||
roles = row.get("feature.semantic.roles") or None
|
||||
def_use = row.get("feature.semantic.def_use") or None
|
||||
obligations = row.get("feature.semantic.obligations")
|
||||
function = row.get("feature.source.function")
|
||||
out: dict[str, Any] = {key: row.get(key) for key in KEYS}
|
||||
out.update(
|
||||
{
|
||||
"diff.distance": row.get("feature.diff.distance"),
|
||||
"diff.on_changed_line": None
|
||||
if row.get("feature.diff.on_changed_line") is None
|
||||
else int(bool(row["feature.diff.on_changed_line"])),
|
||||
"diff.hunk_count": _count(hunks),
|
||||
"diff.hunk_lines": None if hunks is None else sum(end - start + 1 for start, end in hunks),
|
||||
"source.span_lines": None if not span else span[1] - span[0] + 1,
|
||||
"source.is_module": None if function is None else int(function == "<module>"),
|
||||
"source.caller_count": _count(callers),
|
||||
"source.caller_files": None if callers is None else len({path for path, _ in callers}),
|
||||
"source.entry_point_count": _count(row.get("feature.source.entry_points")),
|
||||
"semantic.sources": None if roles is None else sum(1 for role in roles.values() if role == "source"),
|
||||
"semantic.sinks": None if roles is None else sum(1 for role in roles.values() if role == "sink"),
|
||||
"semantic.sanitizers": None if roles is None else sum(1 for role in roles.values() if role == "sanitizer"),
|
||||
"semantic.binds": None if def_use is None else len(def_use.get("binds", [])),
|
||||
"semantic.calls": None if def_use is None else len(def_use.get("calls", [])),
|
||||
"semantic.obligations": _count(obligations),
|
||||
"semantic.obligations_undischarged": None if obligations is None else sum(1 for o in obligations if not o.get("discharged")),
|
||||
}
|
||||
)
|
||||
for producer in ("diff", "source", "semantic", "taxonomy"):
|
||||
out[f"ok.{producer}"] = row.get(f"provenance.{producer}.ok")
|
||||
return out
|
||||
|
||||
|
||||
def feature_frame(table: FeatureTable | Sequence[Mapping[str, Any]]) -> Any:
|
||||
"""The engineered frame: one row per candidate, key and label columns, numeric features, provenance flags."""
|
||||
import pandas as pd
|
||||
|
||||
rows = table.rows if isinstance(table, FeatureTable) else list(table)
|
||||
return pd.DataFrame([_engineer(row) for row in rows])
|
||||
|
||||
|
||||
def features_of(annotation: str) -> list[str]:
|
||||
"""The engineered features that come from one annotation (what an ablation mask on it also drops)."""
|
||||
return [name for name, (_, input_name) in ENGINEERED.items() if input_name == annotation]
|
||||
|
||||
|
||||
def annotations_of(mask: Iterable[str]) -> set[str]:
|
||||
"""Annotations an ablation mask removes, whether it names a producer, a column or an input."""
|
||||
dropped = set(mask)
|
||||
out = set()
|
||||
for name, (column, input_name) in ENGINEERED.items():
|
||||
producer = column.split(".")[1]
|
||||
if input_name in dropped or column in dropped or producer in dropped or name in dropped:
|
||||
out.add(input_name)
|
||||
return out
|
||||
|
||||
|
||||
def screen(frame: Any, *, label: str = "label") -> Any:
|
||||
"""Separation of every engineered feature against the label, grouped by the annotation it renders into.
|
||||
|
||||
The caller reads it two ways: an annotation with no feature above 0.5 carries no signal the model could use
|
||||
from this column alone, and an annotation near 1.0 on training data is a leak to audit before it is trusted."""
|
||||
import pandas as pd
|
||||
|
||||
from ..stats import separation
|
||||
|
||||
columns = [name for name in ENGINEERED if name in frame.columns]
|
||||
table = separation(frame, label, columns)
|
||||
table["annotation"] = [ENGINEERED[name][1] for name in table["column"]]
|
||||
best = (
|
||||
table.dropna(subset=["auc"])
|
||||
.assign(distance=lambda t: (t["auc"] - 0.5).abs())
|
||||
.sort_values("distance", ascending=False)
|
||||
.groupby("annotation", as_index=False)
|
||||
.first()[["annotation", "column", "auc", "n"]]
|
||||
.rename(columns={"column": "strongest_feature"})
|
||||
)
|
||||
return pd.DataFrame(best).sort_values("annotation").reset_index(drop=True)
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Statistics the research side reports with: intervals, paired comparisons, grouped folds, feature separation.
|
||||
|
||||
One place, one convention. Wilson intervals come from statsmodels, exact McNemar from statsmodels, the bootstrap
|
||||
is a repository-clustered percentile bootstrap with linear-interpolation quantiles, folds come from scikit-learn's
|
||||
grouped splitters, and separation is the area under the ROC curve of one column against the label. The libraries
|
||||
are imported inside the functions so the core install (PyYAML only) never pays for them; the `[hunt]` extra does.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
RESAMPLES = 2000
|
||||
ALPHA = 0.05
|
||||
|
||||
|
||||
def wilson(successes: int, n: int, *, alpha: float = ALPHA) -> tuple[float, float] | None:
|
||||
"""The Wilson score interval of a proportion; None when there is no denominator."""
|
||||
if n <= 0:
|
||||
return None
|
||||
from statsmodels.stats.proportion import proportion_confint
|
||||
|
||||
low, high = proportion_confint(successes, n, alpha=alpha, method="wilson")
|
||||
return float(low), float(high)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class McNemar:
|
||||
"""Exact McNemar on paired binary outcomes: `b` units where only A succeeds, `c` where only B does."""
|
||||
|
||||
b: int
|
||||
c: int
|
||||
concordant: int
|
||||
p_value: float
|
||||
|
||||
|
||||
def mcnemar_exact(a: Sequence[bool], b: Sequence[bool]) -> McNemar:
|
||||
"""Two-sided exact McNemar (binomial on the discordant pairs); p = 1 when nothing is discordant."""
|
||||
if len(a) != len(b):
|
||||
raise ValueError("paired outcomes must have the same length")
|
||||
only_a = sum(1 for x, y in zip(a, b, strict=True) if x and not y)
|
||||
only_b = sum(1 for x, y in zip(a, b, strict=True) if y and not x)
|
||||
concordant = len(a) - only_a - only_b
|
||||
if only_a + only_b == 0:
|
||||
return McNemar(only_a, only_b, concordant, 1.0)
|
||||
from statsmodels.stats.contingency_tables import mcnemar
|
||||
|
||||
result = mcnemar([[concordant, only_a], [only_b, 0]], exact=True)
|
||||
return McNemar(only_a, only_b, concordant, float(min(1.0, result.pvalue)))
|
||||
|
||||
|
||||
def cluster_resample(groups: Sequence[str], rng: Any) -> list[tuple[int, int]]:
|
||||
"""Draw groups with replacement; return (draw, row index) pairs so a repeated group keeps its own identity."""
|
||||
names = sorted(set(groups))
|
||||
members: dict[str, list[int]] = {}
|
||||
for index, group in enumerate(groups):
|
||||
members.setdefault(group, []).append(index)
|
||||
out: list[tuple[int, int]] = []
|
||||
for draw, name in enumerate(rng.choice(names, size=len(names), replace=True)):
|
||||
out.extend((draw, index) for index in members[name])
|
||||
return out
|
||||
|
||||
|
||||
def _quantiles(values: Sequence[float], alpha: float) -> tuple[float, float]:
|
||||
import numpy as np
|
||||
|
||||
low, high = np.quantile(np.asarray(values, dtype=float), [alpha / 2, 1 - alpha / 2])
|
||||
return float(low), float(high)
|
||||
|
||||
|
||||
def cluster_bootstrap(
|
||||
frame: Any, group: str, stat: Callable[[Any], float | None], *, resamples: int = RESAMPLES, seed: int = 0, alpha: float = ALPHA
|
||||
) -> tuple[float, float] | None:
|
||||
"""Percentile interval of `stat(frame)` under a bootstrap that resamples whole groups (repositories)."""
|
||||
import numpy as np
|
||||
|
||||
if len(frame) == 0:
|
||||
return None
|
||||
rng = np.random.default_rng(seed)
|
||||
groups = list(frame[group].astype(str))
|
||||
values = []
|
||||
for _ in range(resamples):
|
||||
rows = cluster_resample(groups, rng)
|
||||
sample = frame.iloc[[index for _, index in rows]].copy()
|
||||
sample[group] = [f"{draw}:{groups[index]}" for draw, index in rows]
|
||||
value = stat(sample)
|
||||
if value is not None:
|
||||
values.append(float(value))
|
||||
return _quantiles(values, alpha) if len(values) >= 2 else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Estimate:
|
||||
"""A paired comparison B minus A over units, with its clustered-bootstrap interval."""
|
||||
|
||||
a: float | None
|
||||
b: float | None
|
||||
diff: float | None
|
||||
ci: tuple[float, float] | None
|
||||
se: float | None
|
||||
units: int
|
||||
groups: int
|
||||
resamples: int
|
||||
|
||||
|
||||
def paired_bootstrap(
|
||||
frame: Any,
|
||||
group: str,
|
||||
a: str,
|
||||
b: str,
|
||||
*,
|
||||
stat: Callable[[Any], float | None] | None = None,
|
||||
resamples: int = RESAMPLES,
|
||||
seed: int = 0,
|
||||
alpha: float = ALPHA,
|
||||
) -> Estimate:
|
||||
"""B minus A where columns `a` and `b` hold each unit's outcome in both arms; `stat` reduces a column
|
||||
(mean by default) and may return None when a resample cannot be scored."""
|
||||
import numpy as np
|
||||
|
||||
def mean(column: Any) -> float | None:
|
||||
return None if len(column) == 0 else float(column.mean())
|
||||
|
||||
reduce = stat or mean
|
||||
a_all, b_all = reduce(frame[a]), reduce(frame[b])
|
||||
diff = None if a_all is None or b_all is None else b_all - a_all
|
||||
groups = list(frame[group].astype(str))
|
||||
rng = np.random.default_rng(seed)
|
||||
values = []
|
||||
for _ in range(resamples if groups else 0):
|
||||
rows = [index for _, index in cluster_resample(groups, rng)]
|
||||
sample = frame.iloc[rows]
|
||||
x, y = reduce(sample[a]), reduce(sample[b])
|
||||
if x is not None and y is not None:
|
||||
values.append(y - x)
|
||||
ci = se = None
|
||||
if len(values) >= 2:
|
||||
ci = _quantiles(values, alpha)
|
||||
se = float(np.std(values, ddof=1))
|
||||
return Estimate(a_all, b_all, diff, ci, se, len(frame), len(set(groups)), len(values))
|
||||
|
||||
|
||||
def group_folds(frame: Any, group: str, *, k: int = 5, seed: int = 0) -> list[tuple[list[int], list[int]]]:
|
||||
"""K folds that never split a group across train and test (scikit-learn's GroupKFold, shuffled)."""
|
||||
from sklearn.model_selection import GroupKFold
|
||||
|
||||
groups = list(frame[group].astype(str))
|
||||
if len(set(groups)) < k:
|
||||
raise ValueError(f"{len(set(groups))} groups cannot make {k} folds")
|
||||
splitter = GroupKFold(n_splits=k, shuffle=True, random_state=seed)
|
||||
return [(list(map(int, train)), list(map(int, test))) for train, test in splitter.split(frame, groups=groups)]
|
||||
|
||||
|
||||
def leave_one_group_out(frame: Any, group: str) -> list[tuple[list[int], list[int]]]:
|
||||
from sklearn.model_selection import LeaveOneGroupOut
|
||||
|
||||
groups = list(frame[group].astype(str))
|
||||
return [(list(map(int, train)), list(map(int, test))) for train, test in LeaveOneGroupOut().split(frame, groups=groups)]
|
||||
|
||||
|
||||
def separation(frame: Any, label: str, columns: Iterable[str]) -> Any:
|
||||
"""How well each numeric column alone separates label 1 from label 0: area under the ROC curve, the number
|
||||
of rows it was computed on, and a note when it could not be. 0.5 is no signal; near 0 or 1 is a leak or a
|
||||
strong feature, which the caller decides."""
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
rows = []
|
||||
labelled = frame[frame[label].isin([0, 1])]
|
||||
for column in columns:
|
||||
series = pd.to_numeric(labelled[column], errors="coerce") if column in labelled else None
|
||||
if series is None:
|
||||
rows.append({"column": column, "auc": None, "n": 0, "note": "missing"})
|
||||
continue
|
||||
mask = series.notna()
|
||||
y = labelled.loc[mask, label].astype(int)
|
||||
if mask.sum() == 0 or y.nunique() < 2:
|
||||
rows.append({"column": column, "auc": None, "n": int(mask.sum()), "note": "one class or empty"})
|
||||
continue
|
||||
if series[mask].nunique() < 2:
|
||||
rows.append({"column": column, "auc": 0.5, "n": int(mask.sum()), "note": "constant"})
|
||||
continue
|
||||
rows.append({"column": column, "auc": float(roc_auc_score(y, series[mask])), "n": int(mask.sum()), "note": ""})
|
||||
return pd.DataFrame(rows, columns=["column", "auc", "n", "note"])
|
||||
Reference in New Issue
Block a user