fix(learn): fixed-side negatives only where the fixed side declares the function

A pair now gives a fixed-side negative only when the fixed side's source declares the labelled function (the excerpt builder's matcher; a name on a code line where the matcher resolves neither side), or when a deterministic rename at a distinct fix commit replaces it (one new function, same parameter list, changed body: provenance fixed_side_moved). Anything else is counted under pair_fixed_side.absent_fixed_side, not labelled. 108 pair labels lose their negative: 80 seeded pointer pairs whose fixed side is a guard helper of the same commit, and 28 Real-Vuln-Benchmark traps whose fixed side is a sibling handler of the same snapshot (4 of those handlers are labelled positives themselves). The memory build no longer substitutes the catalog's fix_function on the fixed side. The pair cache is an allowed label-source tree so pointer sides are read through the guard. Snapshot: benchmarks/measurements/2026-10-01-decision-engine-labels (counts only). Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
2026-10-16 03:49:26 +00:00 · 2026-10-01 11:03:54 +02:00
parent 7f6bbacb6e
commit 658f2d5e39
5 changed files with 711 additions and 19 deletions
@@ -247,11 +247,34 @@ def test_repository_groups_merge_urls_forks_and_advisories() -> None:
    assert L.advisories("fix for cve-2024-0001 and GHSA-mh7h-gpmx-fggj") == {"CVE-2024-0001", "GHSA-MH7H-GPMX-FGGJ"}


-def _pair(name: str, *, tier: str = "advisory", unscorable: str | None = None, function: str | None = "f", repo: str = "") -> PairCase:
+VULN_PY = "def f(q):\n    return db.execute('SELECT ' + q)\n"
+FIXED_PY = "def f(q):\n    return db.execute('SELECT ?', (q,))\n"
+
+
+def _pair(
+    name: str,
+    *,
+    tier: str = "advisory",
+    unscorable: str | None = None,
+    function: str | None = "f",
+    repo: str = "",
+    root: Path | None = None,
+    vuln: str = VULN_PY,
+    fixed: str = FIXED_PY,
+    parent: str = "a" * 40,
+    commit: str = "b" * 40,
+) -> PairCase:
    expected = (ExpectedFinding(cwe="CWE-89", vulnerability_class="sqli", path="a.py", evidence="e", function=function),)
+    folder = (root or Path("/nonexistent")) / "pairs" / name
+    if root is not None:
+        folder.mkdir(parents=True, exist_ok=True)
+        header = f"# Provenance: fixture\n# commit: {commit}\n# parent: {parent}\n# function: {function}\n"
+        (folder / "vuln.py").write_text(header + vuln)
+        (folder / "fixed.py").write_text(header + fixed)
    return PairCase(
-        name=name, slice="vfc", language="python", origin="o", vuln_file=Path("v"), fixed_file=Path("f"), relpath="a.py",
-        expected=expected, min_recall=1.0, fix_policy="p", repo=repo, cve="CVE-2024-0001", review_tier=tier, unscorable=unscorable,
+        name=name, slice="vfc", language="python", origin="o", vuln_file=folder / "vuln.py", fixed_file=folder / "fixed.py",
+        relpath="a.py", expected=expected, min_recall=1.0, fix_policy="p", repo=repo, commit=commit, cve="CVE-2024-0001",
+        review_tier=tier, unscorable=unscorable,
    )  # fmt: skip


@@ -262,8 +285,9 @@ def test_pairs_exclude_unscorable_and_title_and_group_by_repository(tmp_path: Pa
        '[[source]]\nid = "pairs"\nkind = "pairs"\nfile = "pairs/catalog.toml"\nrecorded = "x"\nstatus = "x"\nexclude_tiers = ["title"]\n'
    )
    cases = (
-        _pair("ok", repo="https://github.com/o/r"), _pair("twin", unscorable="identical_twin"), _pair("titled", tier="title"),
-        _pair("nofn", function=None), _pair("fork", repo="https://github.com/x/r-fork"),
+        _pair("ok", repo="https://github.com/o/r", root=tmp_path), _pair("twin", unscorable="identical_twin"),
+        _pair("titled", tier="title", root=tmp_path), _pair("nofn", function=None),
+        _pair("fork", repo="https://github.com/x/r-fork", root=tmp_path),
    )  # fmt: skip
    monkeypatch.setattr("openultrasast.pairs.load_pair_catalog", lambda path: cases)
    build, record = L.build_labels(tmp_path, sources_path=tmp_path / "s.toml", cache=tmp_path, assumed_benign=False)
@@ -283,6 +307,78 @@ def test_pairs_exclude_unscorable_and_title_and_group_by_repository(tmp_path: Pa
    assert "titled" in {r.source_ref for r in titled.labels}


+def _pair_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *cases: PairCase) -> tuple[L.Build, dict[str, Any]]:
+    (tmp_path / "pairs").mkdir(exist_ok=True)
+    (tmp_path / "pairs" / "catalog.toml").write_text("")
+    source = '[[source]]\nid = "pairs"\nkind = "pairs"\nfile = "pairs/catalog.toml"\nrecorded = "x"\nstatus = "x"\n'
+    (tmp_path / "s.toml").write_text(source)
+    monkeypatch.setattr("openultrasast.pairs.load_pair_catalog", lambda path: cases)
+    return L.build_labels(tmp_path, sources_path=tmp_path / "s.toml", cache=tmp_path, assumed_benign=False)
+
+
+def _fixed_rows(build: L.Build, name: str) -> list[L.Label]:
+    return [r for r in build.labels if r.source_ref == name and r.pin_role == "fixed"]
+
+
+GATE = "def _gate(q):\n    return bool(q)\n"  # a guard helper of the same snapshot, same parameter list as the labelled f
+
+
+def test_a_fixed_side_without_the_function_gives_no_negative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    snapshot = "c" * 40
+    cases = (
+        _pair("gate", root=tmp_path, fixed=GATE, parent=snapshot, commit=snapshot),  # the kolega-ai-dev pointer shape
+        _pair("other", root=tmp_path, fixed="def g(a, b):\n    return a\n"),  # a real fix, but no renamed equivalent
+        _pair("unread", root=None),
+    )
+    build, record = _pair_build(tmp_path, monkeypatch, *cases)
+    for name in ("gate", "other", "unread"):
+        assert _fixed_rows(build, name) == [], name
+        assert {(r.unit, r.label) for r in build.labels if r.source_ref == name} == {("pin", 1), ("delta", 1)}, "positives stay"
+    assert record["pair_fixed_side"] == {"absent_fixed_side": {"labels": 3, "by_family": {"injection": 3}}, "fixed_side_moved": 0}
+    assert record["skipped"] == {
+        f"pairs: no fixed-side negative ({L.ABSENT_SAME_SNAPSHOT})": 1,
+        f"pairs: no fixed-side negative ({L.ABSENT_NO_EQUIVALENT})": 1,
+        f"pairs: no fixed-side negative ({L.ABSENT_UNREAD})": 1,
+    }
+    assert record["verified_pin_labels"]["by_family"]["injection"]["negative_groups"] == 0
+
+
+def test_a_renamed_function_at_a_real_fix_is_a_moved_negative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    moved = "def f_safe(q):\n    return db.execute('SELECT ?', (q,))\n"
+    build, record = _pair_build(tmp_path, monkeypatch, _pair("moved", root=tmp_path, fixed=moved, repo="https://github.com/o/r"))
+    rows = _fixed_rows(build, "moved")
+    assert {(r.unit, r.label, r.candidate, r.provenance) for r in rows} == {
+        ("pin", 0, "a.py::f_safe", L.FIXED_MOVED),
+        ("delta", 0, "a.py::f_safe", L.FIXED_MOVED),
+    }
+    assert all("f -> f_safe" in r.evidence for r in rows)
+    assert record["pair_fixed_side"]["fixed_side_moved"] == 1 and record["pair_fixed_side"]["absent_fixed_side"]["labels"] == 0
+    # the same rename in one snapshot (no fix commit) is a sibling function, never a negative
+    snapshot = "d" * 40
+    same, _ = _pair_build(tmp_path, monkeypatch, _pair("sibling", root=tmp_path, fixed=moved, parent=snapshot, commit=snapshot))
+    assert _fixed_rows(same, "sibling") == []
+    # two candidate renames are ambiguous: no negative
+    two = moved + "\n\ndef f_other(q):\n    return q\n"
+    ambiguous, _ = _pair_build(tmp_path, monkeypatch, _pair("two", root=tmp_path, fixed=two))
+    assert _fixed_rows(ambiguous, "two") == []
+
+
+def test_ordinary_pairs_and_matcher_limits_keep_their_negative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    js_vuln, js_fixed = (
+        "module.exports = {\n  ps: function(pid) {\n    exec('ps ' + pid)\n  }\n}\n",
+        ("module.exports = {\n  ps: function(pid) {\n    execFile('ps', [pid])\n  }\n}\n"),
+    )
+    cache = tmp_path / "pointer-cache"
+    monkeypatch.setenv("OPENULTRASAST_PAIR_CACHE", str(cache))
+    js = _pair("js", root=cache, function="ps", vuln=js_vuln, fixed=js_fixed)
+    js = PairCase(**{**js.__dict__, "language": "javascript"})
+    build, record = _pair_build(tmp_path, monkeypatch, _pair("plain", root=tmp_path), js)
+    assert {(r.unit, r.candidate, r.provenance) for r in _fixed_rows(build, "plain")} == {("pin", "a.py::f", ""), ("delta", "a.py::f", "")}
+    assert {r.candidate for r in _fixed_rows(build, "js")} == {"a.py::ps"}, "the pointer file in the pair cache was read"
+    assert record["pair_fixed_side"]["absent_fixed_side"]["labels"] == 0
+    assert record["verified_pin_labels"]["by_family"]["injection"]["negative_rows"] == 2
+
+
 def test_the_snapshot_carries_counts_only(tmp_path: Path) -> None:
    _, record, _, pins = _build(tmp_path)
    text = json.dumps(record)