fix(learn): withhold label-tracking signals from the v1 classifier; leak audit

Maintainer decision "Neutralise both for v1".

1. Phantom candidates. A pair side's candidates are now only the functions that
   side declares (plane/harvest.py `declares`, the excerpt builder's
   function_bounds); a labelled function absent from a side is listed under
   `absent_on_side` and never asked, and a side declaring none of its verify
   family's functions is no verify unit. The recorded harvest units held 58
   phantom verify candidates (53 fixed, 5 vulnerable sides). build_features.py
   skips label rows whose side does not declare the function and drops their
   earlier rows (MemoryStore.drop_rows); rebuilt the pair records from the
   recorded signals, no model call: 126 label rows absent_on_side (117 fixed,
   9 vulnerable) -- access_control 13, config_secrets 4, deserialization 20,
   injection 28, memory 4, output_encoding 16, path 15, untrusted_destination
   26 -- so 252 feature rows dropped (1712 -> 1460); no other row changed.

2. Engine withheld. Input profiles in learn/schema.py (`v1` withholds the
   engine instrument, `full` withholds nothing); ProgramSpec.inputs defaults to
   v1 and compile.toml records it. Neither the engine's state nor its values
   reach the candidate, examples, demonstrations or the instruction proposer's
   error summary, and the Gower retrieval distance leaves the engine block out.
   Stored records are unchanged. design.md Risks records the decision and that
   the engine returns only through a pre-registered A/B on assumed-benign and
   benign-control negatives.

3. `ousast learn audit-leaks` (learn/leaks.py): per feature and per instrument
   state, separation = (pairs the signal alone puts on the right side - on the
   wrong side) / pairs, over all pairs and per family; flagged at >= 0.20 over
   >= 10 pairs. Counts under
   benchmarks/measurements/2026-10-01-decision-engine-leak-audit/ (before and
   after the rebuild). After: facts.function_lines (0.80 untrusted_destination,
   0.75 config_secrets, 0.26 injection; higher on fixed), verify.turns_mean
   0.67 config_secrets, eng.* 0.35-0.40 output_encoding, verify.flag_a/votes/
   site_offset and agree.final 0.20-0.30 on 10-12 pair families. Before the
   rebuild the phantom fixed sides drove state.source/state.roles/ms.flagged
   to 1.0 and state.engine to 0.95 on deserialization.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_learn_leaks.py

@@ -0,0 +1,94 @@
+"""The leakage audit (``ousast learn audit-leaks``): a signal that alone names the side of a pair is flagged, one that
+does not is not; rows pair by case, candidate and family; an empty store is an error, never a clean audit."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from learn_fixtures import x_record
+
+from openultrasast.cli import main
+from openultrasast.learn.leaks import Pair, Signal, audit, pair_rows, separation
+from openultrasast.learn.schema import BY_NAME
+from openultrasast.plane.memory import FileStore
+
+
+def _pairs(n: int = 20) -> list[Pair]:
+    """``n`` pairs: the engine state tracks the side (ran on the vulnerable side, failed on the fixed one) in all but
+    two; quick hits differ in half the pairs, in alternating directions (no information)."""
+    out = []
+    for i in range(n):
+        leaks = i >= 2
+        vulnerable = x_record(hits=(i % 2) * 2, engine="ran", findings=1)
+        fixed = x_record(hits=((i + 1) % 2) * 2, engine="failed" if leaks else "ran", findings=1)
+        out.append(Pair("injection", vulnerable, fixed))
+    return out
+
+
+def test_a_leaking_signal_is_flagged_and_a_clean_one_is_not() -> None:
+    report = audit(_pairs(), "static")
+    assert report["pairs"] == {"all": 20, "injection": 20}
+    engine = report["signals"]["state.engine"]["all"]
+    assert engine == {"pairs": 20, "differ": 18, "correct": 18, "wrong": 0, "separation": 0.9}
+    assert "state.engine" in report["flagged_signals"] and "eng.findings" in report["flagged_signals"]
+    hits = report["signals"]["qr.enabled_hits"]["all"]
+    assert hits["differ"] == 20 and hits["correct"] == hits["wrong"] == 10 and hits["separation"] == 0.0
+    assert "qr.enabled_hits" not in report["flagged_signals"] and "state.quick" not in report["flagged_signals"]
+    assert report["flagged"][0]["separation"] == 0.9 and all(c["separation"] >= 0.2 for c in report["flagged"])
+    few = audit(_pairs()[:9], "static")  # under min_pairs: never flagged, however clean the split
+    assert few["flagged"] == [] and few["signals"]["state.engine"]["all"]["separation"] > 0.7
+
+
+def test_an_ordered_signal_has_one_direction() -> None:
+    spec = BY_NAME["qr.enabled_hits"]
+    signal = Signal(spec.name, spec.instrument, True, spec)
+    higher = [Pair("f", x_record(hits=3), x_record(hits=1)) for _ in range(6)] + [Pair("f", x_record(hits=0), x_record(hits=2))] * 2
+    cell = separation(signal, higher)
+    assert cell["direction"] == "higher on vulnerable" and (cell["correct"], cell["wrong"]) == (6, 2) and cell["separation"] == 0.5
+
+
+def _row(task: str, family: str = "injection", candidate: str = "a.py::f", **kw: object) -> dict:
+    rec = x_record(**kw)  # type: ignore[arg-type]
+    pin = ("0" if task.endswith("v") else "1") * 40
+    return {"id": f"{task}:{candidate}", "kind": "features", "repo": "github.com/o/r", "pin": pin, "run": "harvest", "task": task,
+            "population": "pairs", "split": "pairs", "image": "none", **rec}  # fmt: skip
+
+
+def test_rows_pair_by_case_candidate_and_family() -> None:
+    sides = {"u1v": ("case-1", "vulnerable"), "u1f": ("case-1", "fixed"), "u2v": ("case-2", "vulnerable")}
+    rows = [_row("u1v"), _row("u1f", engine="failed"), _row("u2v"), _row("u9")]
+    pairs, skipped = pair_rows(rows, sides, "static")
+    assert len(pairs) == 1 and pairs[0].vulnerable["instruments"]["engine"]["state"] == "ran"
+    assert pairs[0].fixed["instruments"]["engine"]["state"] == "failed"
+    assert skipped == {"no side": 1, "one side only": 1}
+    assert pair_rows(rows, sides, "plane")[0] == []  # another profile's rows are not read
+
+
+def test_cli_audits_the_store_and_refuses_an_empty_one(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+    store = FileStore(tmp_path / "memory")
+    units = tmp_path / "units.json"
+    sides = {}
+    rows = []
+    for i in range(12):
+        sides[f"c{i}v"], sides[f"c{i}f"] = {"ref": f"case-{i}", "side": "vulnerable"}, {"ref": f"case-{i}", "side": "fixed"}
+        rows += [_row(f"c{i}v", findings=1), _row(f"c{i}f", engine="failed")]
+    store.put_rows(rows)
+    units.write_text(json.dumps(sides))
+    out = tmp_path / "audit.json"
+    args = ["learn", "audit-leaks", "--memory", f"file://{tmp_path / 'memory'}", "--units", str(units), "--profile", "static"]
+    assert main([*args, "--out", str(out)]) == 0
+    report = json.loads(out.read_text())
+    assert report["rows_read"] == 24 and report["pairs"]["all"] == 12 and "state.engine" in report["flagged_signals"]
+    assert "a.py" not in out.read_text()  # counts only: no candidate reaches the report
+    empty = ["learn", "audit-leaks", "--memory", f"file://{tmp_path / 'empty'}", "--units", str(units)]
+    assert main(empty) == 2 and "no pair to audit" in capsys.readouterr().err
+
+
+def test_drop_rows_removes_only_the_named_rows(tmp_path: Path) -> None:
+    store = FileStore(tmp_path / "memory")
+    store.put_rows([_row("u1v"), _row("u1v", candidate="a.py::g")])
+    assert store.drop_rows("github.com/o/r", "0" * 40, ["u1v:a.py::g", "absent"]) == 1
+    assert [r.row["id"] for r in store.rows(kind="features")] == ["u1v:a.py::f"]
+    assert store.drop_rows("github.com/o/r", "0" * 40, ["u1v:a.py::f"]) == 1 and store.rows(kind="features") == []

tests/test_learn_program.py

@@ -75,11 +75,42 @@ def test_prompts_hold_no_excluded_field_and_no_candidate_label() -> None:
 
 
 def test_missing_instruments_are_said_in_words() -> None:
-    prompt, _ = program().prepare(candidate(engine="none"), FOLD)
+    prompt, _ = program(inputs="full").prepare(candidate(engine="none"), FOLD)
     tail = str(prompt.messages[-1]["content"]).rsplit("Candidate:\n", 1)[1]
     assert "engine: no coverage" in tail and "delta: not applicable" in tail and "eng.findings" not in tail
 
 
+ENGINE_FIELD = re.compile(r"^(engine:|eng\.)", re.MULTILINE)
+
+
+@pytest.mark.parametrize("engine", ["ran", "failed", "none"])
+def test_v1_prompts_contain_no_engine_field(engine: str) -> None:
+    """v1 (the default input profile) withholds the engine -- state and values -- from every rendered message: the
+    candidate, the retrieved examples and the demonstrations, as the scripted client receives them."""
+    assert ProgramSpec().inputs == "v1"
+    demo_example = EXAMPLES[0]
+    demo = Demo(demo_example.id, demo_example.excerpt_sha, 0, "injection", "not_vulnerable", 0.8, "Line 11 escapes q.", (11,))
+    chat = ScriptedChat()
+    caller = Caller(MeteredClient(chat, prices=PRICES), "deepseek-flash", PRICES)
+    program(demos=(demo,), k=2).decide(candidate(engine=engine, findings=2 if engine == "ran" else 0), FOLD, caller)
+    assert len(chat.calls) == 2
+    for call in chat.calls:
+        text = "\n".join(str(m["content"]) for m in call["messages"])
+        assert "Demonstration 1:" in text and text.count("(label: ") == 6 and "Candidate:" in text
+        assert not ENGINE_FIELD.search(text), ENGINE_FIELD.search(text)
+        assert "qr.enabled_hits:" in text and "delta: not applicable" in text  # the other instruments still show
+    full, _ = program(demos=(demo,), inputs="full").prepare(candidate(engine=engine), FOLD)
+    assert ENGINE_FIELD.search("\n".join(str(m["content"]) for m in full.messages))
+
+
+def test_v1_proposer_summary_contains_no_engine_field() -> None:
+    from openultrasast.learn.compile import _signal_summary
+
+    leaky = replace(EXAMPLES[0], **{k: v for k, v in x_record(hits=2, findings=3).items() if k in ("x", "instruments")})
+    assert not any(k.startswith("eng.") for k in _signal_summary(leaky))
+    assert any(k.startswith("eng.") for k in _signal_summary(leaky, "full"))
+
+
 def test_the_prefix_is_byte_identical_across_candidates() -> None:
     demo_example = EXAMPLES[0]
     demo = Demo(demo_example.id, demo_example.excerpt_sha, 0, "injection", "not_vulnerable", 0.8, "Line 11 escapes q.", (11,))

tests/test_learn_retrieve.py

@@ -30,6 +30,8 @@ QUICK_ALL = [s for s in features_for("static") if s.instrument == "quick"]
 
 
 def _d(a: dict, b: dict, **kw: str) -> float:
+    """The full distance (every block, the engine included): the definition. v1 withholds the engine (below)."""
+    kw.setdefault("inputs", "full")
     return gower(a["x"], a["instruments"], b["x"], b["instruments"], "static", **kw)
 
 
@@ -53,6 +55,22 @@ def test_prior_features_are_dropped_with_priors_off() -> None:
     assert _d(plain, prior, priors="on") == pytest.approx(5 / 10 / len(QUICK_ALL) / BLOCKS)
 
 
+def test_v1_distance_contains_no_engine_field() -> None:
+    """v1 (the default) withholds the engine: neither its state nor its values move the distance, so neighbours cannot
+    carry it; every other block still counts, over one block fewer."""
+    ran, failed, none = x_record(findings=3), x_record(engine="failed"), x_record(engine="none")
+    for a, b in ((ran, failed), (ran, none), (failed, none), (x_record(findings=0), ran)):
+        assert gower(a["x"], a["instruments"], b["x"], b["instruments"], "static") == 0.0
+        assert _d(a, b) > 0.0
+    hits = x_record(hits=10, engine="failed")
+    expected = (10 / 20 + 10 / 10) / len(QUICK_NO_PRIOR) / (BLOCKS - 1)
+    assert gower(ran["x"], ran["instruments"], hits["x"], hits["instruments"], "static") == pytest.approx(expected)
+    same, other = example("same-engine", group="g1", label=0, engine="failed"), example("other-engine", group="g2", label=1, findings=4)
+    v1 = [gower(failed["x"], failed["instruments"], e.x, e.instruments, "static") for e in (same, other)]
+    full = [gower(failed["x"], failed["instruments"], e.x, e.instruments, "static", inputs="full") for e in (same, other)]
+    assert v1[0] == v1[1] and full[0] < full[1]  # the engine state ranked them; in v1 it cannot
+
+
 def _target(**kw: object) -> Target:
     rec = x_record(**kw)  # type: ignore[arg-type]
     return Target("target/repo", "injection", "static", rec["x"], rec["instruments"])

tests/test_plane_harvest.py

@@ -5,7 +5,9 @@ from __future__ import annotations
 import json
 import subprocess
 from pathlib import Path
+from types import SimpleNamespace
 
+import pytest
 import yaml
 
 from openultrasast.plane import reconciler
@@ -98,3 +100,28 @@ def test_blob_sha1_is_gits_and_declaration_lines() -> None:
     lines = text.splitlines()
     assert declaration_line(lines, "m.py", "b") == 4
     assert declaration_line(lines, "m.py", "missing") == 1 and declaration_line(None, "m.py", "a") == 1
+
+
+def test_pair_sides_ask_only_the_functions_they_declare(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """A fixed side whose fix removed a labelled function never gets it as a candidate (no phantom verify candidate);
+    it is listed under ``absent_on_side``. A side declaring none of its verify family's functions is no verify unit."""
+    from openultrasast import pairs
+    from openultrasast.plane.harvest import pair_units
+
+    vuln, fixed = tmp_path / "vuln.py", tmp_path / "fixed.py"
+    vuln.write_text("import os\n\ndef run(cmd):\n    os.system(cmd)\n\ndef keep(x):\n    return x\n")
+    fixed.write_text("import shlex\n\ndef keep(x):\n    return shlex.quote(x)\n")
+    case = SimpleNamespace(name="p-cwe-078", vuln_file=vuln, fixed_file=fixed, relpath="app.py", context_files=(), repo="", slice="s")
+    monkeypatch.setattr(pairs, "load_pair_catalog", lambda _catalog: [case])
+    rows = [{"source": "pairs", "unit": "pin", "source_ref": "p-cwe-078", "family": "injection", "candidate": f"app.py::{fn}"}
+            for fn in ("run", "keep")]  # fmt: skip
+    labels = tmp_path / "labels.jsonl"
+    labels.write_text("".join(json.dumps(r) + "\n" for r in rows))
+    by_side = {u.side: u for u in pair_units(labels, tmp_path / "catalog.toml")}
+    assert [c[1] for c in by_side["vulnerable"].candidates] == ["keep", "run"] and by_side["vulnerable"].absent_on_side == ()
+    assert [c[1] for c in by_side["fixed"].candidates] == ["keep"] and [c[1] for c in by_side["fixed"].verify] == ["keep"]
+    assert by_side["fixed"].absent_on_side == ("app.py::run",)
+    assert by_side["fixed"].index_row()["absent_on_side"] == ["app.py::run"]
+    fixed.write_text("import shlex\n\ndef other(x):\n    return shlex.quote(x)\n")
+    gone = next(u for u in pair_units(labels, tmp_path / "catalog.toml") if u.side == "fixed")
+    assert gone.candidates == () and gone.verify == () and gone.family == "" and len(gone.absent_on_side) == 2