mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Fable 5.1
d9d6335ab3
hunter-learning-loop groups 1-3 (first increment). The feature table is the contract: one row per touched function with diff, source, semantic and taxonomy columns, provenance per producer, labels by the benchmark's rule. Producers read the repository at head only (a planted advisory id and commit text never reach a feature). The hunt program is a DSPy ReAct loop over a signature whose inputs are the unmasked columns in a fixed order; answers outside the closed family list or citing unseen lines score as unknown. The runner writes records the benchmark scorer reads, resumable by change, program id and mask, and refuses the held-out pilot unless the run is a report run. Research extra [hunt] = dspy, pandas; the core install is unchanged. 43 tests on a scripted model, no network. Co-Authored-By: Claude Fable 5.1 <[email protected]>
98 lines
4.6 KiB
Python
98 lines
4.6 KiB
Python
"""The hunt program with a scripted model: signature masks, rendering, the answer rules, the program id."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
dspy = pytest.importorskip("dspy")
|
|
from dspy.utils import DummyLM # noqa: E402
|
|
|
|
from openultrasast.hunt import program as hp # noqa: E402
|
|
|
|
CHANGE = {"id": "c1", "kind": "vulnerability", "path": "app.py", "function": "second", "family": "injection"}
|
|
ROWS = [
|
|
{
|
|
"change_id": "c1",
|
|
"path": "app.py",
|
|
"line": 5,
|
|
"feature.diff.hunks": [[3, 3]],
|
|
"feature.source.function": "second",
|
|
"feature.source.span": [5, 7],
|
|
"feature.source.callers": [["web.py", 2]],
|
|
"feature.source.entry_points": [],
|
|
"feature.semantic.roles": {"eval": "sink"},
|
|
"feature.semantic.def_use": {"binds": [], "calls": [{"line": 6, "name": "eval"}]},
|
|
"feature.semantic.obligations": [],
|
|
"feature.taxonomy.families": [{"id": "injection", "definition": "untrusted data reaches an interpreter"}],
|
|
}
|
|
]
|
|
TEXT = {"app.py": ["def first():", " v = input()", " return v", "", "def second():", " return eval(first())", ""]}
|
|
|
|
|
|
def test_signature_drops_masked_inputs_and_keeps_the_order():
|
|
full = hp.signature_for()
|
|
assert list(full.input_fields) == [name for name, _, _ in hp.INPUTS]
|
|
masked = hp.signature_for(["semantic", "callers"])
|
|
assert "roles" not in masked.input_fields and "def_use" not in masked.input_fields and "callers" not in masked.input_fields
|
|
assert list(masked.input_fields) == ["change_summary", "hunks", "function_text", "entry_points", "families"]
|
|
assert "findings" in masked.output_fields
|
|
|
|
|
|
def test_render_rows_numbers_the_function_and_omits_masked_inputs():
|
|
inputs = hp.render_rows(CHANGE, ROWS, TEXT)
|
|
assert inputs["change_summary"] == "app.py::second"
|
|
assert json.loads(inputs["hunks"]) == {"app.py": [[3, 3]]}
|
|
assert " 5 def second():" in inputs["function_text"] and " 6 " in inputs["function_text"]
|
|
assert "input()" not in inputs["function_text"] # only the span is shown
|
|
assert inputs["families"].startswith("injection: ")
|
|
masked = hp.render_rows(CHANGE, ROWS, TEXT, mask=["diff", "function_text"])
|
|
assert "hunks" not in masked and "function_text" not in masked and "roles" in masked
|
|
assert hp.shown_lines(inputs) == {"app.py": {5, 6, 7}}
|
|
|
|
|
|
def test_check_turns_off_list_families_and_unseen_citations_into_unknown():
|
|
inputs = hp.render_rows(CHANGE, ROWS, TEXT)
|
|
good = {"path": "app.py", "line": 6, "family": "injection", "confidence": 0.9, "rationale": "eval", "cited_lines": [6]}
|
|
bad_family = {**good, "family": "sql-parser"}
|
|
bad_cite = {**good, "cited_lines": [2]}
|
|
out = hp.check([good, bad_family, bad_cite], inputs)
|
|
assert out[0]["family"] == "injection" and "unknown_reason" not in out[0]
|
|
assert out[1]["family"] == "unknown" and out[1]["reported_family"] == "sql-parser" and out[1]["unknown_reason"] == "family_not_in_list"
|
|
assert out[2]["family"] == "unknown" and out[2]["unknown_reason"] == "cited_line_not_shown"
|
|
|
|
|
|
def test_program_without_tools_runs_on_a_scripted_model_and_has_a_stable_id():
|
|
answer = [{"path": "app.py", "line": 6, "family": "injection", "confidence": 0.8, "rationale": "eval of input", "cited_lines": [6]}]
|
|
lm = DummyLM([{"findings": json.dumps(answer)}, {"findings": json.dumps(answer)}])
|
|
dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
|
|
program = hp.HuntProgram(mask=["obligations"])
|
|
inputs = hp.render_rows(CHANGE, ROWS, TEXT, mask=program.mask)
|
|
prediction = program(**inputs)
|
|
assert prediction.findings == [{**answer[0]}]
|
|
assert prediction.findings[0]["family"] == "injection"
|
|
assert hp.program_id(program, mask=program.mask) == hp.program_id(hp.HuntProgram(mask=["obligations"]), mask=["obligations"])
|
|
assert hp.program_id(program) != hp.program_id(program, mask=["obligations"])
|
|
|
|
|
|
def test_tools_are_clamped_to_the_checkout(tmp_path):
|
|
(tmp_path / "a.py").write_text("x = 1\n")
|
|
tools = {tool.name: tool for tool in hp.repository_tools(tmp_path)}
|
|
assert set(tools) == {"read_file", "grep_repo", "find_refs"}
|
|
assert "x = 1" in tools["read_file"](path="a.py")
|
|
from openultrasast.hunter_tools import PathEscapesRepo
|
|
|
|
with pytest.raises(PathEscapesRepo):
|
|
tools["read_file"](path="../../etc/passwd")
|
|
assert (
|
|
"(no matches)" in tools["grep_repo"](pattern="nothing_here_\\d+")
|
|
or tools["grep_repo"](pattern="nothing_here_\\d+") == "(no matches)"
|
|
)
|
|
|
|
|
|
def test_build_lm_needs_a_key_from_the_environment(monkeypatch):
|
|
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
|
with pytest.raises(RuntimeError, match="DEEPSEEK_API_KEY"):
|
|
hp.build_lm()
|