feat(stats): one statistics module on scipy, statsmodels and scikit-learn; engineered features tied to the annotations

openultrasast.stats: Wilson intervals (statsmodels), exact McNemar, a repository-clustered percentile bootstrap and its paired form with one quantile convention, grouped folds (GroupKFold, leave-one-group-out), and per-column separation (ROC AUC) for the leak audit and the signal screen. Checked against the benchmark scorer's Wilson and the learn harness's McNemar. hunt/features.py turns the feature table into the engineered frame (distances, counts, flags) and keeps the map from every engineered feature to the annotation the model is shown, so an ablation mask drops both, and screen() says which annotation carries signal before a model call is spent. The [hunt] extra now carries the libraries; the core install is unchanged. The learn harness's own copies stay until that line is revived. Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 03:49:26 +00:00 · 2026-10-13 12:43:02 +02:00
parent d9d6335ab3
commit c95b9241d8
5 changed files with 521 additions and 1 deletions
@@ -0,0 +1,101 @@
+"""Engineered features from the feature table, their tie to the annotations, and the separation screen."""
+
+from __future__ import annotations
+
+import pytest
+
+pd = pytest.importorskip("pandas")
+pytest.importorskip("sklearn")
+
+from openultrasast.hunt import features  # noqa: E402
+
+ROW = {
+    "change_id": "c1",
+    "repo": "r",
+    "head": "h",
+    "path": "app.py",
+    "line": 5,
+    "kind": "vulnerability",
+    "fold": 1,
+    "label": 1,
+    "family": "injection",
+    "feature.diff.hunks": [[3, 3], [10, 12]],
+    "feature.diff.on_changed_line": False,
+    "feature.diff.distance": 2,
+    "feature.source.function": "second",
+    "feature.source.span": [5, 7],
+    "feature.source.callers": [["web.py", 2], ["web.py", 9], ["cli.py", 1]],
+    "feature.source.entry_points": [{"path": "web.py", "line": 1, "kind": "route"}],
+    "feature.source.language": "python",
+    "feature.semantic.roles": {"input": "source", "eval": "sink", "escape": "sanitizer", "x": "sink"},
+    "feature.semantic.def_use": {"binds": [{"line": 6}], "calls": [{"line": 6}, {"line": 7}]},
+    "feature.semantic.obligations": [
+        {"operation": "auth", "line": 6, "discharged": False},
+        {"operation": "csrf", "line": 7, "discharged": True},
+    ],
+    "feature.taxonomy.families": [{"id": "injection", "definition": "x"}],
+    "provenance.diff.ok": True,
+    "provenance.source.ok": True,
+    "provenance.semantic.ok": True,
+    "provenance.taxonomy.ok": True,
+}
+
+
+def test_feature_frame_engineers_counts_distances_and_flags():
+    absent = {
+        **ROW,
+        "change_id": "c2",
+        "label": 0,
+        "feature.source.function": "<module>",
+        "feature.source.span": None,
+        "feature.semantic.roles": None,
+        "feature.semantic.def_use": None,
+        "feature.semantic.obligations": None,
+        "provenance.semantic.ok": False,
+    }
+    frame = features.feature_frame([ROW, absent])
+    first = frame.iloc[0]
+    assert first["diff.distance"] == 2 and first["diff.on_changed_line"] == 0
+    assert first["diff.hunk_count"] == 2 and first["diff.hunk_lines"] == 4
+    assert first["source.span_lines"] == 3 and first["source.is_module"] == 0
+    assert first["source.caller_count"] == 3 and first["source.caller_files"] == 2 and first["source.entry_point_count"] == 1
+    assert (first["semantic.sources"], first["semantic.sinks"], first["semantic.sanitizers"]) == (1, 2, 1)
+    assert first["semantic.binds"] == 1 and first["semantic.calls"] == 2
+    assert first["semantic.obligations"] == 2 and first["semantic.obligations_undischarged"] == 1
+    second = frame.iloc[1]
+    assert second["source.is_module"] == 1 and pd.isna(second["source.span_lines"])
+    assert pd.isna(second["semantic.sinks"]) and second["ok.semantic"] is False or second["ok.semantic"] == False  # noqa: E712
+    assert list(frame["label"]) == [1, 0]
+
+
+def test_every_engineered_feature_maps_to_an_annotation_and_masks_follow():
+    from openultrasast.hunt.program import INPUTS
+
+    inputs = {name for name, _, _ in INPUTS}
+    assert all(annotation in inputs for _, annotation in features.ENGINEERED.values())
+    assert set(features.features_of("roles")) == {"semantic.sources", "semantic.sinks", "semantic.sanitizers"}
+    assert features.annotations_of(["semantic"]) == {"roles", "def_use", "obligations"}
+    assert features.annotations_of(["feature.source.callers"]) == {"callers"}
+    assert features.annotations_of(["hunks"]) == {"hunks"}
+    assert features.annotations_of([]) == set()
+
+
+def test_screen_reports_the_strongest_feature_per_annotation():
+    rows = []
+    for i in range(12):
+        label = i % 2
+        rows.append(
+            {
+                **ROW,
+                "change_id": f"c{i}",
+                "label": label,
+                "feature.diff.distance": 0 if label else 30,
+                "feature.source.callers": [["a.py", 1]] * (3 if label else 1),
+                "feature.semantic.roles": {"eval": "sink"},
+            }
+        )
+    frame = features.feature_frame(rows)
+    out = features.screen(frame).set_index("annotation")
+    assert out.loc["hunks", "strongest_feature"] == "diff.distance" and out.loc["hunks", "auc"] == 0.0
+    assert out.loc["callers", "strongest_feature"] == "source.caller_count" and out.loc["callers", "auc"] == 1.0
+    assert out.loc["roles", "auc"] == 0.5  # constant: no signal
@@ -0,0 +1,104 @@
+"""The shared statistics against the formulas the benchmark and the learn harness used."""
+
+from __future__ import annotations
+
+import math
+
+import pytest
+
+pd = pytest.importorskip("pandas")
+pytest.importorskip("statsmodels")
+pytest.importorskip("sklearn")
+
+from openultrasast import stats  # noqa: E402
+
+
+def test_wilson_matches_the_benchmark_scorer_to_three_decimals():
+    from benchmarks.unseen.score import wilson as scorer_wilson
+
+    for successes, n in ((6, 12), (5, 12), (130, 239), (1, 149), (0, 12), (221, 239)):
+        low, high = stats.wilson(successes, n)
+        old_low, old_high = scorer_wilson(successes, n)
+        assert abs(low - old_low) < 1e-3 and abs(high - old_high) < 1e-3, (successes, n)
+    assert stats.wilson(0, 0) is None
+    low, high = stats.wilson(6, 12)
+    assert round(low, 3) == 0.254 and round(high, 3) == 0.746  # the pilot's hunter recall interval
+
+
+def test_exact_mcnemar_matches_the_learn_harness():
+    import random
+
+    from openultrasast.learn.experiments import exact_mcnemar
+
+    rng = random.Random(1)
+    for _ in range(20):
+        n = rng.randint(1, 40)
+        a = [rng.random() < 0.5 for _ in range(n)]
+        b = [rng.random() < 0.5 for _ in range(n)]
+        ours = stats.mcnemar_exact(a, b)
+        theirs = exact_mcnemar(list(zip(a, b, strict=True)))
+        assert (ours.b, ours.c, ours.concordant) == (theirs.b, theirs.c, theirs.concordant)
+        assert math.isclose(ours.p_value, theirs.p_value, abs_tol=1e-9), (a, b)
+    assert stats.mcnemar_exact([True, False], [True, False]).p_value == 1.0
+    with pytest.raises(ValueError):
+        stats.mcnemar_exact([True], [True, False])
+
+
+def _frame(seed=0, groups=8, per=6):
+    import numpy as np
+
+    rng = np.random.default_rng(seed)
+    rows = []
+    for g in range(groups):
+        for _ in range(per):
+            a = rng.random() < 0.4
+            rows.append({"repo": f"r{g}", "a": int(a), "b": int(a or rng.random() < 0.3), "label": int(rng.random() < 0.5)})
+    return pd.DataFrame(rows)
+
+
+def test_cluster_bootstrap_is_seeded_and_brackets_the_estimate():
+    frame = _frame()
+    stat = lambda f: float(f["a"].mean())  # noqa: E731
+    first = stats.cluster_bootstrap(frame, "repo", stat, resamples=300, seed=3)
+    second = stats.cluster_bootstrap(frame, "repo", stat, resamples=300, seed=3)
+    assert first == second and first[0] <= stat(frame) <= first[1]
+    assert stats.cluster_bootstrap(frame.iloc[0:0], "repo", stat) is None
+
+
+def test_paired_bootstrap_reports_b_minus_a_with_an_interval():
+    frame = _frame()
+    estimate = stats.paired_bootstrap(frame, "repo", "a", "b", resamples=300, seed=1)
+    assert estimate.diff == pytest.approx(frame["b"].mean() - frame["a"].mean())
+    assert estimate.ci is not None and estimate.ci[0] <= estimate.diff <= estimate.ci[1]
+    assert estimate.units == len(frame) and estimate.groups == 8 and estimate.resamples == 300
+    assert estimate.se is not None and estimate.se > 0
+
+
+def test_group_folds_never_split_a_repository():
+    frame = _frame(groups=10)
+    folds = stats.group_folds(frame, "repo", k=5, seed=0)
+    assert len(folds) == 5
+    for train, test in folds:
+        assert not set(frame.iloc[train]["repo"]) & set(frame.iloc[test]["repo"])
+        assert sorted(train + test) == list(range(len(frame)))
+    assert len(stats.leave_one_group_out(frame, "repo")) == 10
+    with pytest.raises(ValueError):
+        stats.group_folds(_frame(groups=3), "repo", k=5)
+
+
+def test_separation_scores_each_column_alone():
+    frame = pd.DataFrame(
+        {
+            "label": [1, 1, 1, 0, 0, 0, None],
+            "perfect": [5, 6, 7, 1, 2, 3, 9],
+            "noise": [1, 1, 1, 1, 1, 1, 1],
+            "inverse": [1, 2, 3, 5, 6, 7, 0],
+            "sparse": [None, None, None, None, None, None, 1],
+        }
+    )
+    out = stats.separation(frame, "label", ["perfect", "noise", "inverse", "sparse", "missing"]).set_index("column")
+    assert out.loc["perfect", "auc"] == 1.0 and out.loc["perfect", "n"] == 6
+    assert out.loc["inverse", "auc"] == 0.0
+    assert out.loc["noise", "auc"] == 0.5 and out.loc["noise", "note"] == "constant"
+    assert out.loc["sparse", "auc"] is None or pd.isna(out.loc["sparse", "auc"])
+    assert out.loc["missing", "note"] == "missing"