feat(hunt): feature table, producers, labels, metric, artifact, the DSPy hunt program, runner and CLI

hunter-learning-loop groups 1-3 (first increment). The feature table is the
contract: one row per touched function with diff, source, semantic and
taxonomy columns, provenance per producer, labels by the benchmark's rule.
Producers read the repository at head only (a planted advisory id and
commit text never reach a feature). The hunt program is a DSPy ReAct loop
over a signature whose inputs are the unmasked columns in a fixed order;
answers outside the closed family list or citing unseen lines score as
unknown. The runner writes records the benchmark scorer reads, resumable
by change, program id and mask, and refuses the held-out pilot unless the
run is a report run. Research extra [hunt] = dspy, pandas; the core
install is unchanged. 43 tests on a scripted model, no network.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_hunt_artifact.py

@@ -0,0 +1,49 @@
+"""Checks for stable run records and budget accounting."""
+
+import json
+
+import pytest
+
+from openultrasast.hunt.artifact import RunArtifact, UsageLedger
+
+
+def artifact():
+    return RunArtifact(program_id="program", schema_version=1, mask=("source",), data_digests={"train": "abc"},
+                       configurations=[{"seed": 0}], usage={"usd": 0.1, "calls": 1, "replay_hits": 0, "wall_seconds": 2},
+                       versions={"dspy": "3.4.0", "litellm": "1", "openultrasast": "2.0.1"}, created="2026-10-13T00:00:00Z")
+
+
+def test_artifact_round_trip_and_tamper(tmp_path):
+    run = artifact()
+    assert run.artifact_id == artifact().artifact_id
+    path = run.write(tmp_path)
+    assert RunArtifact.read(path) == run
+    payload = json.loads(path.read_text())
+    payload["program_id"] = "tampered"
+    path.write_text(json.dumps(payload))
+    with pytest.raises(ValueError, match="id"):
+        RunArtifact.read(path)
+
+
+def test_ledger_and_budget_prevent_artifact(tmp_path):
+    run = artifact()
+    ledger = UsageLedger(artifact=run)
+    ledger.add({"usd": 0.25, "calls": 1, "wall_seconds": 2})
+    ledger.add({"usd": 0, "replay_hits": 1, "wall_seconds": 1})
+    assert ledger.usage == {"usd": 0.25, "calls": 1, "replay_hits": 1, "wall_seconds": 3}
+    assert ledger.exhausted({"usd": 0.25, "calls": 10})
+    assert ledger.exhausted({"usd": 10, "calls": 1})
+    assert ledger.finish({"usd": 0.25}) is None
+    assert not list(tmp_path.iterdir())
+    result = ledger.finish({"usd": 1, "calls": 2})
+    assert result is run
+    assert result.usage == ledger.usage
+    assert run.finish({"usd": 0.25}) is None
+    assert run.write(tmp_path, budget={"usd": 0.25}) is None
+    assert not list(tmp_path.iterdir())
+
+
+@pytest.mark.parametrize("value", [-1, float("nan"), float("inf")])
+def test_invalid_usage_rejected(value):
+    with pytest.raises(ValueError):
+        UsageLedger().add({"usd": value})

tests/test_hunt_labels.py

@@ -0,0 +1,47 @@
+"""Checks for fix-site and ordinary labels."""
+
+import pytest
+
+from openultrasast.hunt.labels import label_rows
+from openultrasast.hunt.table import FeatureTable
+
+
+def positive():
+    return {"kind": "vulnerability", "path": "app.py", "function": "handle", "function_lines": [10, 20],
+            "family": "injection", "case_id": "verified-case"}
+
+
+def test_labels_match_name_or_span_on_same_path():
+    rows = [{"path": "app.py", "function": "handle", "line": 1},
+            {"path": "app.py", "function": "nested", "line": 12},
+            {"path": "other.py", "function": "handle", "line": 12},
+            {"path": "app.py", "function": "other", "line": 30}]
+    labelled = label_rows(rows, positive())
+    assert [r["label"] for r in labelled] == [1, 1, None, None]
+    for data in labelled[:2]:
+        assert data["family"] == "injection"
+        assert data["fix_span"] == [10, 20]
+        assert data["label_ref"] == "verified-case"
+        assert data["verification"] is None
+    assert "label" not in rows[0]
+
+
+def test_feature_function_and_ordinary_rows():
+    rows = [{"path": "app.py", "line": 1, "feature.source.function": "handle"}]
+    assert label_rows(rows, positive())[0]["label"] == 1
+    ordinary = label_rows([{"path": "app.py", "line": 1}], {"kind": "ordinary", "case_id": "draw"})
+    assert ordinary[0]["label"] == 0
+    assert len(FeatureTable(ordinary).labelled().rows) == 1
+
+
+@pytest.mark.parametrize("kind", [None, "fix", "unknown"])
+def test_unknown_change_kind_rejected(kind):
+    with pytest.raises(ValueError):
+        label_rows([], {"kind": kind})
+
+
+def test_positive_requires_traceable_reference():
+    change = positive()
+    del change["case_id"]
+    with pytest.raises(ValueError, match="case_id"):
+        label_rows([], change)

tests/test_hunt_metric.py

@@ -0,0 +1,46 @@
+"""Checks that search scoring follows the benchmark's location rule."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from benchmarks.unseen.score import catches
+from openultrasast.hunt.metric import search_metric, search_outcome
+
+
+@pytest.mark.parametrize("change", [
+    {"family": "injection", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
+    {"family": "path", "path": "other.py", "function": "read", "function_lines": [100, 140]},
+    {"family": "config_secrets", "path": "app.py", "function": "handler", "function_lines": [100, 140]},
+])
+def test_matches_score_catches(change):
+    change = {**change, "kind": "vulnerability", "files": [{"path": "app.py", "head_lines": [[20, 22]]}]}
+    for path in ("app.py", "other.py"):
+        for line in (9, 10, 22, 32, 33, 110, 500):
+            for function in ("handler", "other"):
+                for family in (change["family"], "unknown"):
+                    finding = {"path": path, "line": line, "function": function, "family": family}
+                    record = {**change, "push": {"admission": {"defects": [
+                        {**finding, "locations": [[path, line]]}
+                    ]}}}
+                    assert search_outcome(change, [finding]).hit == bool(catches(record))
+
+
+def test_counts_scope_feedback_and_tolerance():
+    change = {"kind": "vulnerability", "path": "app.py", "function": "handle", "family": "path",
+              "function_lines": [20, 22], "files": [{"path": "app.py", "head_lines": [[20, 22]]}]}
+    findings = [{"path": "app.py", "line": 10, "family": "path"},
+                {"path": "other.py", "line": 30, "family": "path"}]
+    outcome = search_outcome(change, findings)
+    assert (outcome.hit, outcome.suspicions, outcome.on_diff) == (True, 2, 1)
+    assert "other.py:30" in outcome.feedback
+    missed = search_outcome(change, findings, tolerance=0)
+    assert not missed.hit
+    assert "missed app.py:20 handle family path" in missed.feedback
+    assert search_metric({"change": change}, {"findings": findings}) == 1.0
+    assert search_metric(SimpleNamespace(change=change), SimpleNamespace(findings=[])) == 0.0
+
+
+@pytest.mark.parametrize("count,expected", [(0, 1.0), (1, 0.8), (5, 0.0), (7, 0.0)])
+def test_ordinary_metric(count, expected):
+    assert search_metric({"kind": "ordinary", "files": []}, {"findings": [{}] * count}) == expected

tests/test_hunt_producers.py

@@ -0,0 +1,158 @@
+"""Checks for repository-only annotations on real local git changes."""
+
+import json
+import subprocess
+
+import pytest
+
+from openultrasast import mapping, source
+from openultrasast.hunt import producers as module
+from openultrasast.hunt.producers import PRODUCERS, Context, candidate_rows, compute_rows
+from openultrasast.hunt.table import FeatureTable
+from openultrasast.model.taxonomy import FAMILY_IDS
+
+
+@pytest.fixture
+def repository(tmp_path):
+    def git(*args):
+        return subprocess.run(["git", "-C", str(tmp_path), *args], check=True, capture_output=True, text=True).stdout.strip()
+
+    git("init", "-q")
+    git("config", "user.name", "Fixture")
+    git("config", "user.email", "[email protected]")
+    initial = ('def first():\n    value = input()\n    return value\n\n'
+               'def second():\n    value = first()\n    return value\n')
+    (tmp_path / "app.py").write_text(initial)
+    (tmp_path / "notes.txt").write_text("GHSA-xxxx-xxxx-xxxx CVE-2024-0001\n")
+    (tmp_path / "unknown.zzz").write_text("opaque\n")
+    git("add", "app.py", "notes.txt", "unknown.zzz")
+    git("commit", "-qm", "initial")
+    base = git("rev-parse", "HEAD")
+    (tmp_path / "app.py").write_text(initial.replace("return value", "return eval(value)", 1))
+    (tmp_path / "unknown.zzz").write_text("opaque changed\n")
+    git("add", "app.py", "unknown.zzz")
+    git("commit", "-qm", "fix XSS CVE-2024-0001")
+    head = git("rev-parse", "HEAD")
+    data = {"id": "change", "repo": "fixture/repo", "base": base, "head": head, "kind": "vulnerability", "fold": 1,
+            "path": "app.py", "function": "second", "function_lines": [5, 7], "family": "injection", "case_id": "verified",
+            "advisory": "GHSA-xxxx-xxxx-xxxx", "commit_message": "fix XSS CVE-2024-0001",
+            "files": [{"path": "app.py", "head_lines": [[3, 3]]}]}
+    return tmp_path, data
+
+
+def test_every_column_candidates_and_cached_work(repository, monkeypatch):
+    root, change = repository
+    calls = []
+    real = mapping.analyze_entry_points
+
+    def entries(*args):
+        calls.append(1)
+        return real(*args)
+
+    monkeypatch.setattr(mapping, "analyze_entry_points", entries)
+    candidates = candidate_rows(root, change)
+    assert [r["line"] for r in candidates] == [1, 5]
+    rows = compute_rows(root, change)
+    assert len(calls) == 1
+    table = FeatureTable(rows)
+    assert len(table.feature_columns()) == 12
+    assert [r["label"] for r in rows] == [None, 1]
+    first, second = rows
+    assert first["feature.diff.hunks"] == [[3, 3]]
+    assert first["feature.diff.distance"] == 2
+    assert not first["feature.diff.on_changed_line"]
+    assert first["feature.source.function"] == "first"
+    assert first["feature.source.span"] == [1, 3]
+    assert first["feature.source.callers"] == [["app.py", 6]]
+    assert first["feature.source.language"] == "python"
+    assert isinstance(first["feature.source.entry_points"], list)
+    assert first["feature.semantic.roles"]["input"] == "source"
+    assert first["feature.semantic.roles"]["eval"] == "sink"
+    assert first["feature.semantic.def_use"]["binds"] == [{"line": 2, "name": "value", "names": ["input"], "call_name": "input"}]
+    assert {c["name"] for c in first["feature.semantic.def_use"]["calls"]} == {"input", "eval"}
+    assert isinstance(first["feature.semantic.obligations"], list)
+    assert [f["id"] for f in first["feature.taxonomy.families"]] == list(FAMILY_IDS)
+    assert second["feature.diff.distance"] == 2
+    for row in rows:
+        for producer in PRODUCERS:
+            assert set(producer.columns) <= set(row)
+            assert row[f"provenance.{producer.name}.ok"] is True
+            assert row[f"provenance.{producer.name}.reason"] is None
+            assert row[f"provenance.{producer.name}.version"]
+            assert row[f"provenance.{producer.name}.bytes_read"] > 0
+    ctx = Context(root, change)
+    diff = PRODUCERS[0]
+    assert diff.compute(root, change, "app.py", 3, ctx)["feature.diff.on_changed_line"]
+    assert diff.compute(root, change, "notes.txt", 1, ctx)["feature.diff.distance"] is None
+
+
+def test_unsupported_language_is_absent_with_reason(repository):
+    root, change = repository
+    change = {**change, "kind": "ordinary", "files": [{"path": "unknown.zzz", "head_lines": [[1, 1]]}]}
+    rows = compute_rows(root, change)
+    assert len(rows) == 1
+    assert rows[0]["provenance.semantic.ok"] is False
+    assert rows[0]["provenance.semantic.reason"] == "language_unsupported"
+    assert all(rows[0][column] is None for column in PRODUCERS[2].columns if column.startswith("feature."))
+    assert rows[0]["label"] == 0
+    FeatureTable(rows)
+
+
+def test_planted_advisory_and_commit_text_never_enter_features(repository):
+    root, change = repository
+    rows = compute_rows(root, change)
+    values = json.dumps([{k: v for k, v in row.items() if k.startswith("feature.")} for row in rows])
+    for planted in ("GHSA-xxxx-xxxx-xxxx", "CVE-2024-0001", "fix XSS"):
+        assert planted not in values
+
+
+def test_producer_exceptions_are_type_only(repository, monkeypatch):
+    root, change = repository
+
+    def failed(*args, **kwargs):
+        raise RuntimeError("secret credential")
+
+    monkeypatch.setattr(source, "facts_for", failed)
+    row = compute_rows(root, change)[0]
+    assert row["provenance.source.reason"] == "RuntimeError"
+    assert row["provenance.source.ok"] is False
+    assert all(row[c] is None for c in PRODUCERS[1].columns if c.startswith("feature."))
+    assert "secret credential" not in json.dumps(row)
+
+
+def test_range_rejection_retains_row_and_reason(repository, monkeypatch):
+    root, change = repository
+
+    def failed(*args):
+        raise ValueError("private text")
+
+    monkeypatch.setattr(source, "ranges", failed)
+    rows = compute_rows(root, {**change, "kind": "ordinary"})
+    assert len(rows) == 1
+    assert rows[0]["provenance.source.reason"] == "ValueError"
+    assert rows[0]["provenance.source.ok"] is False
+
+
+def test_private_or_outside_paths_are_not_read(repository):
+    root, change = repository
+    for path in (".env", "../outside.py"):
+        with pytest.raises(ValueError):
+            candidate_rows(root, {**change, "files": [{"path": path, "head_lines": [[1, 1]]}]})
+
+
+def test_head_mismatch_is_not_a_clean_measurement(repository):
+    root, change = repository
+    rows = compute_rows(root, {**change, "head": change["base"], "kind": "ordinary"})
+    assert rows[0]["provenance.diff.ok"] is False
+    assert rows[0]["provenance.diff.reason"] == "head_mismatch"
+
+
+def test_obligation_result_uses_dominance_not_absence_of_findings(repository, monkeypatch):
+    root, change = repository
+    text = ('from flask import request\n@app.route("/item")\ndef first():\n'
+            '    return User.query.get(request.args["id"])\n')
+    (root / "app.py").write_text(text)
+    monkeypatch.setattr(module, "_git", lambda *args: change["head"])
+    row = compute_rows(root, {**change, "kind": "ordinary", "files": [{"path": "app.py", "head_lines": [[4, 4]]}]})[0]
+    assert row["feature.semantic.obligations"]
+    assert row["feature.semantic.obligations"][0]["discharged"] is False

tests/test_hunt_program.py

@@ -0,0 +1,97 @@
+"""The hunt program with a scripted model: signature masks, rendering, the answer rules, the program id."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+dspy = pytest.importorskip("dspy")
+from dspy.utils import DummyLM  # noqa: E402
+
+from openultrasast.hunt import program as hp  # noqa: E402
+
+CHANGE = {"id": "c1", "kind": "vulnerability", "path": "app.py", "function": "second", "family": "injection"}
+ROWS = [
+    {
+        "change_id": "c1",
+        "path": "app.py",
+        "line": 5,
+        "feature.diff.hunks": [[3, 3]],
+        "feature.source.function": "second",
+        "feature.source.span": [5, 7],
+        "feature.source.callers": [["web.py", 2]],
+        "feature.source.entry_points": [],
+        "feature.semantic.roles": {"eval": "sink"},
+        "feature.semantic.def_use": {"binds": [], "calls": [{"line": 6, "name": "eval"}]},
+        "feature.semantic.obligations": [],
+        "feature.taxonomy.families": [{"id": "injection", "definition": "untrusted data reaches an interpreter"}],
+    }
+]
+TEXT = {"app.py": ["def first():", "    v = input()", "    return v", "", "def second():", "    return eval(first())", ""]}
+
+
+def test_signature_drops_masked_inputs_and_keeps_the_order():
+    full = hp.signature_for()
+    assert list(full.input_fields) == [name for name, _, _ in hp.INPUTS]
+    masked = hp.signature_for(["semantic", "callers"])
+    assert "roles" not in masked.input_fields and "def_use" not in masked.input_fields and "callers" not in masked.input_fields
+    assert list(masked.input_fields) == ["change_summary", "hunks", "function_text", "entry_points", "families"]
+    assert "findings" in masked.output_fields
+
+
+def test_render_rows_numbers_the_function_and_omits_masked_inputs():
+    inputs = hp.render_rows(CHANGE, ROWS, TEXT)
+    assert inputs["change_summary"] == "app.py::second"
+    assert json.loads(inputs["hunks"]) == {"app.py": [[3, 3]]}
+    assert "    5  def second():" in inputs["function_text"] and "    6  " in inputs["function_text"]
+    assert "input()" not in inputs["function_text"]  # only the span is shown
+    assert inputs["families"].startswith("injection: ")
+    masked = hp.render_rows(CHANGE, ROWS, TEXT, mask=["diff", "function_text"])
+    assert "hunks" not in masked and "function_text" not in masked and "roles" in masked
+    assert hp.shown_lines(inputs) == {"app.py": {5, 6, 7}}
+
+
+def test_check_turns_off_list_families_and_unseen_citations_into_unknown():
+    inputs = hp.render_rows(CHANGE, ROWS, TEXT)
+    good = {"path": "app.py", "line": 6, "family": "injection", "confidence": 0.9, "rationale": "eval", "cited_lines": [6]}
+    bad_family = {**good, "family": "sql-parser"}
+    bad_cite = {**good, "cited_lines": [2]}
+    out = hp.check([good, bad_family, bad_cite], inputs)
+    assert out[0]["family"] == "injection" and "unknown_reason" not in out[0]
+    assert out[1]["family"] == "unknown" and out[1]["reported_family"] == "sql-parser" and out[1]["unknown_reason"] == "family_not_in_list"
+    assert out[2]["family"] == "unknown" and out[2]["unknown_reason"] == "cited_line_not_shown"
+
+
+def test_program_without_tools_runs_on_a_scripted_model_and_has_a_stable_id():
+    answer = [{"path": "app.py", "line": 6, "family": "injection", "confidence": 0.8, "rationale": "eval of input", "cited_lines": [6]}]
+    lm = DummyLM([{"findings": json.dumps(answer)}, {"findings": json.dumps(answer)}])
+    dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
+    program = hp.HuntProgram(mask=["obligations"])
+    inputs = hp.render_rows(CHANGE, ROWS, TEXT, mask=program.mask)
+    prediction = program(**inputs)
+    assert prediction.findings == [{**answer[0]}]
+    assert prediction.findings[0]["family"] == "injection"
+    assert hp.program_id(program, mask=program.mask) == hp.program_id(hp.HuntProgram(mask=["obligations"]), mask=["obligations"])
+    assert hp.program_id(program) != hp.program_id(program, mask=["obligations"])
+
+
+def test_tools_are_clamped_to_the_checkout(tmp_path):
+    (tmp_path / "a.py").write_text("x = 1\n")
+    tools = {tool.name: tool for tool in hp.repository_tools(tmp_path)}
+    assert set(tools) == {"read_file", "grep_repo", "find_refs"}
+    assert "x = 1" in tools["read_file"](path="a.py")
+    from openultrasast.hunter_tools import PathEscapesRepo
+
+    with pytest.raises(PathEscapesRepo):
+        tools["read_file"](path="../../etc/passwd")
+    assert (
+        "(no matches)" in tools["grep_repo"](pattern="nothing_here_\\d+")
+        or tools["grep_repo"](pattern="nothing_here_\\d+") == "(no matches)"
+    )
+
+
+def test_build_lm_needs_a_key_from_the_environment(monkeypatch):
+    monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
+    with pytest.raises(RuntimeError, match="DEEPSEEK_API_KEY"):
+        hp.build_lm()

tests/test_hunt_runner.py

@@ -0,0 +1,110 @@
+"""The hunt runner over a fixture repository with a scripted program: records, resumability, the pilot refusal."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+
+import pytest
+
+dspy = pytest.importorskip("dspy")
+from dspy.utils import DummyLM  # noqa: E402
+
+from benchmarks.unseen import score  # noqa: E402
+from openultrasast.hunt import program as hp  # noqa: E402
+from openultrasast.hunt import runner  # noqa: E402
+
+
+@pytest.fixture
+def repository(tmp_path):
+    root = tmp_path / "repo"
+    root.mkdir()
+
+    def git(*args):
+        return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip()
+
+    git("init", "-q")
+    git("config", "user.name", "Fixture")
+    git("config", "user.email", "[email protected]")
+    (root / "app.py").write_text(
+        "def first():\n    value = input()\n    return value\n\ndef second():\n    value = first()\n    return value\n"
+    )
+    git("add", "app.py")
+    git("commit", "-qm", "initial")
+    base = git("rev-parse", "HEAD")
+    (root / "app.py").write_text(
+        "def first():\n    value = input()\n    return value\n\ndef second():\n    value = first()\n    return eval(value)\n"
+    )
+    git("add", "app.py")
+    git("commit", "-qm", "change")
+    head = git("rev-parse", "HEAD")
+    positive = {
+        "id": "pos", "repo": "fixture/repo", "base": base, "head": head, "kind": "vulnerability", "case_id": "case-1",
+        "path": "app.py", "function": "second", "function_lines": [5, 7], "family": "injection",
+        "files": [{"path": "app.py", "head_lines": [[7, 7]]}],
+    }  # fmt: skip
+    ordinary = {
+        "id": "ord",
+        "repo": "fixture/repo",
+        "base": base,
+        "head": head,
+        "kind": "ordinary",
+        "case_id": "ord-1",
+        "files": [{"path": "app.py", "head_lines": [[7, 7]]}],
+    }
+    manifest = tmp_path / "pool-fixture.toml"
+    manifest.write_text("status = 'fixture'\n")
+    return root, manifest, [positive, ordinary]
+
+
+def _scripted_program():
+    answer = [{"path": "app.py", "line": 7, "family": "injection", "confidence": 0.9, "rationale": "eval of input", "cited_lines": [7]}]
+    lm = DummyLM([{"findings": json.dumps(answer)}, {"findings": "[]"}, {"findings": json.dumps(answer)}, {"findings": "[]"}])
+    dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
+    return hp.HuntProgram(mask=["obligations"])
+
+
+def test_runner_writes_scorable_records_and_resumes(repository, tmp_path):
+    root, manifest, changes = repository
+    program = _scripted_program()
+    pid = hp.program_id(program, mask=program.mask)
+    out = tmp_path / "out"
+    artifact = runner.run(
+        manifest, {"fixture/repo": str(root)}, out, program=program, program_id=pid, mask=program.mask, load_changes=lambda m, s: changes
+    )
+    assert artifact is not None and artifact.program_id == pid
+    rows = score.load_records(out)
+    assert {r["id"]: r["status"] for r in rows} == {"pos": "hunt_covered", "ord": "hunt_covered"}
+    pos = next(r for r in rows if r["id"] == "pos")
+    assert pos["instrument"]["head_verified"] and pos["instrument"]["hook_bytes_read"] > 0
+    defects = pos["push"]["admission"]["defects"]
+    assert defects and defects[0]["family"] == "injection" and defects[0]["locations"] == [["app.py", 7]]
+    assert defects[0]["program_id"] == pid and defects[0]["mask"] == ["obligations"]
+    assert defects[0]["function"] == "second"
+    assert score.catches(pos)
+    assert "obligations" not in pos["inputs_present"] and "function_text" in pos["inputs_present"]
+    # a second run with the same program and mask touches nothing
+    before = {p.name: p.read_bytes() for p in out.glob("*.json")}
+    runner.run(
+        manifest, {"fixture/repo": str(root)}, out, program=program, program_id=pid, mask=program.mask, load_changes=lambda m, s: changes
+    )
+    assert {p.name: p.read_bytes() for p in out.glob("*.json") if p.name in before} == before
+
+
+def test_runner_refuses_the_pilot_unless_it_is_a_report_run(tmp_path):
+    manifest = tmp_path / "pool-valid.toml"
+    manifest.write_text("status = 'frozen'\n")
+    (tmp_path / "freeze-valid.json").write_text(json.dumps({"freeze_digest": runner.PILOT_FREEZE_DIGEST}))
+    with pytest.raises(ValueError, match="held out"):
+        runner.refuse_pilot(manifest, pilot_report=False)
+    runner.refuse_pilot(manifest, pilot_report=True)
+
+
+def test_instrument_failure_is_recorded_with_a_type_only(repository, tmp_path):
+    root, manifest, changes = repository
+    program = _scripted_program()
+    broken = {**changes[0], "id": "bad", "head": "0" * 40}
+    record = runner.hunt_change(broken, root, program=program, program_id="p", mask=(), render=hp.render_rows, tools_for=None)
+    assert record["status"] == "instrument_failure"
+    assert record["instrument"]["failure"] == "CalledProcessError"
+    assert record["push"]["admission"]["defects"] == []

tests/test_hunt_table.py

@@ -0,0 +1,73 @@
+"""Checks for the versioned hunter table."""
+
+import builtins
+import json
+
+import pytest
+
+from openultrasast.hunt.table import SCHEMA_VERSION, FeatureTable
+
+
+def row(label=1):
+    return {
+        "change_id": "c", "repo": "r", "head": "h", "path": "app.py", "line": 1,
+        "kind": "vulnerability", "fold": 0, "label": label, "family": "injection",
+        "fix_span": [1, 3], "label_ref": "case", "verification": None,
+        "feature.diff.distance": 0, "provenance.diff.ok": True,
+        "provenance.diff.reason": None, "provenance.diff.version": "1", "provenance.diff.bytes_read": 10,
+    }
+
+
+def test_round_trip_digest_and_mask(tmp_path):
+    table = FeatureTable([row(), row(None), row(0)])
+    path = tmp_path / "table.jsonl"
+    table.write_jsonl(path)
+    restored = FeatureTable.read_jsonl(path)
+    assert restored.rows == table.rows
+    assert restored.digest() == table.digest()
+    assert SCHEMA_VERSION == 1
+    assert len(table.labelled().rows) == 2
+    masked = table.mask(["diff"])
+    assert masked.masked == ("diff",)
+    assert masked.feature_columns() == []
+    assert table.feature_columns() == ["feature.diff.distance"]
+    assert masked.digest() != table.digest()
+    masked.write_jsonl(path)
+    assert FeatureTable.read_jsonl(path).masked == ("diff",)
+    assert table.mask(["feature.diff.distance"]).feature_columns() == []
+    assert table.digest() == FeatureTable([dict(reversed(list(row().items()))), row(None), row(0)]).digest()
+
+
+@pytest.mark.parametrize("extra", [{"unknown": 1}, {"feature.diff.typo": 1}, {"provenance.diff.secret": 1}])
+def test_unknown_columns_rejected(extra):
+    with pytest.raises(ValueError):
+        FeatureTable([{**row(), **extra}])
+
+
+def test_missing_provenance_rejected():
+    data = row()
+    del data["provenance.diff.ok"]
+    with pytest.raises(ValueError, match="provenance"):
+        FeatureTable([data])
+
+
+def test_schema_version_and_empty_table(tmp_path):
+    path = tmp_path / "table.jsonl"
+    FeatureTable([]).write_jsonl(path)
+    assert FeatureTable.read_jsonl(path).rows == []
+    path.write_text(json.dumps({"schema_version": 99, "masked": []}) + "\n")
+    with pytest.raises(ValueError, match="schema"):
+        FeatureTable.read_jsonl(path)
+
+
+def test_pandas_is_optional(monkeypatch):
+    real_import = builtins.__import__
+
+    def importing(name, *args, **kwargs):
+        if name == "pandas":
+            raise ImportError("missing")
+        return real_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", importing)
+    with pytest.raises(RuntimeError, match="pandas.*hunt"):
+        FeatureTable([row()]).to_pandas()