feat(hunt): feature table, producers, labels, metric, artifact, the DSPy hunt program, runner and CLI

hunter-learning-loop groups 1-3 (first increment). The feature table is the
contract: one row per touched function with diff, source, semantic and
taxonomy columns, provenance per producer, labels by the benchmark's rule.
Producers read the repository at head only (a planted advisory id and
commit text never reach a feature). The hunt program is a DSPy ReAct loop
over a signature whose inputs are the unmasked columns in a fixed order;
answers outside the closed family list or citing unseen lines score as
unknown. The runner writes records the benchmark scorer reads, resumable
by change, program id and mask, and refuses the held-out pilot unless the
run is a report run. Research extra [hunt] = dspy, pandas; the core
install is unchanged. 43 tests on a scripted model, no network.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_hunt_runner.py

@@ -0,0 +1,110 @@
+"""The hunt runner over a fixture repository with a scripted program: records, resumability, the pilot refusal."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+
+import pytest
+
+dspy = pytest.importorskip("dspy")
+from dspy.utils import DummyLM  # noqa: E402
+
+from benchmarks.unseen import score  # noqa: E402
+from openultrasast.hunt import program as hp  # noqa: E402
+from openultrasast.hunt import runner  # noqa: E402
+
+
+@pytest.fixture
+def repository(tmp_path):
+    root = tmp_path / "repo"
+    root.mkdir()
+
+    def git(*args):
+        return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip()
+
+    git("init", "-q")
+    git("config", "user.name", "Fixture")
+    git("config", "user.email", "[email protected]")
+    (root / "app.py").write_text(
+        "def first():\n    value = input()\n    return value\n\ndef second():\n    value = first()\n    return value\n"
+    )
+    git("add", "app.py")
+    git("commit", "-qm", "initial")
+    base = git("rev-parse", "HEAD")
+    (root / "app.py").write_text(
+        "def first():\n    value = input()\n    return value\n\ndef second():\n    value = first()\n    return eval(value)\n"
+    )
+    git("add", "app.py")
+    git("commit", "-qm", "change")
+    head = git("rev-parse", "HEAD")
+    positive = {
+        "id": "pos", "repo": "fixture/repo", "base": base, "head": head, "kind": "vulnerability", "case_id": "case-1",
+        "path": "app.py", "function": "second", "function_lines": [5, 7], "family": "injection",
+        "files": [{"path": "app.py", "head_lines": [[7, 7]]}],
+    }  # fmt: skip
+    ordinary = {
+        "id": "ord",
+        "repo": "fixture/repo",
+        "base": base,
+        "head": head,
+        "kind": "ordinary",
+        "case_id": "ord-1",
+        "files": [{"path": "app.py", "head_lines": [[7, 7]]}],
+    }
+    manifest = tmp_path / "pool-fixture.toml"
+    manifest.write_text("status = 'fixture'\n")
+    return root, manifest, [positive, ordinary]
+
+
+def _scripted_program():
+    answer = [{"path": "app.py", "line": 7, "family": "injection", "confidence": 0.9, "rationale": "eval of input", "cited_lines": [7]}]
+    lm = DummyLM([{"findings": json.dumps(answer)}, {"findings": "[]"}, {"findings": json.dumps(answer)}, {"findings": "[]"}])
+    dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
+    return hp.HuntProgram(mask=["obligations"])
+
+
+def test_runner_writes_scorable_records_and_resumes(repository, tmp_path):
+    root, manifest, changes = repository
+    program = _scripted_program()
+    pid = hp.program_id(program, mask=program.mask)
+    out = tmp_path / "out"
+    artifact = runner.run(
+        manifest, {"fixture/repo": str(root)}, out, program=program, program_id=pid, mask=program.mask, load_changes=lambda m, s: changes
+    )
+    assert artifact is not None and artifact.program_id == pid
+    rows = score.load_records(out)
+    assert {r["id"]: r["status"] for r in rows} == {"pos": "hunt_covered", "ord": "hunt_covered"}
+    pos = next(r for r in rows if r["id"] == "pos")
+    assert pos["instrument"]["head_verified"] and pos["instrument"]["hook_bytes_read"] > 0
+    defects = pos["push"]["admission"]["defects"]
+    assert defects and defects[0]["family"] == "injection" and defects[0]["locations"] == [["app.py", 7]]
+    assert defects[0]["program_id"] == pid and defects[0]["mask"] == ["obligations"]
+    assert defects[0]["function"] == "second"
+    assert score.catches(pos)
+    assert "obligations" not in pos["inputs_present"] and "function_text" in pos["inputs_present"]
+    # a second run with the same program and mask touches nothing
+    before = {p.name: p.read_bytes() for p in out.glob("*.json")}
+    runner.run(
+        manifest, {"fixture/repo": str(root)}, out, program=program, program_id=pid, mask=program.mask, load_changes=lambda m, s: changes
+    )
+    assert {p.name: p.read_bytes() for p in out.glob("*.json") if p.name in before} == before
+
+
+def test_runner_refuses_the_pilot_unless_it_is_a_report_run(tmp_path):
+    manifest = tmp_path / "pool-valid.toml"
+    manifest.write_text("status = 'frozen'\n")
+    (tmp_path / "freeze-valid.json").write_text(json.dumps({"freeze_digest": runner.PILOT_FREEZE_DIGEST}))
+    with pytest.raises(ValueError, match="held out"):
+        runner.refuse_pilot(manifest, pilot_report=False)
+    runner.refuse_pilot(manifest, pilot_report=True)
+
+
+def test_instrument_failure_is_recorded_with_a_type_only(repository, tmp_path):
+    root, manifest, changes = repository
+    program = _scripted_program()
+    broken = {**changes[0], "id": "bad", "head": "0" * 40}
+    record = runner.hunt_change(broken, root, program=program, program_id="p", mask=(), render=hp.render_rows, tools_for=None)
+    assert record["status"] == "instrument_failure"
+    assert record["instrument"]["failure"] == "CalledProcessError"
+    assert record["push"]["admission"]["defects"] == []