mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(mcp): the findings list carries the rung, and leads with what was decided
The stated final objective is the rung plus an LLM way of diving in without being forced to rethink the code. The second half is largely plumbing, because the material already exists: the witness (`username -> db.session.execute(...) in models/user_model.py (line 73, 8 steps)`) is the reasoning, already written down and already attached to the finding. So this session's recurring shape was worth checking rather than assuming, and it was there. `openultrasast.get_finding` returns the whole record including the witness, but `openultrasast.findings` dropped the rung: an LLM listing libpng's 198 rows could not tell the one the graph decided from the 197 an enumerator proposed without fetching all 198 one at a time. That is exactly the re-derivation the list exists to spare it. The list now carries the rung, orders strongest first, and returns a per-rung tally so a caller can triage before it fetches anything. The brief records what still stands between here and the objective: the witness is one line and the query already walks the path elements before discarding all but the first; the suspicion band will be reasoned about at length by an LLM, which is worse than a human ignoring it; and a family that cannot decide its class must say so, or the model will confidently explain a silence. Gates byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
20edd9354b
commit
d232f03597
@@ -113,3 +113,27 @@ byte-identical through all nine fixes, which is the same statement from the othe
|
|||||||
* **Negative results get recorded as measurements, not as failures.** `c/memory` scoring zero on libpng is a
|
* **Negative results get recorded as measurements, not as failures.** `c/memory` scoring zero on libpng is a
|
||||||
finding about the abstraction, and the artifact says which abstraction would reach it (size arithmetic,
|
finding about the abstraction, and the artifact says which abstraction would reach it (size arithmetic,
|
||||||
i.e. a constraint problem) rather than leaving it as a gap to be tuned at.
|
i.e. a constraint problem) rather than leaving it as a gap to be tuned at.
|
||||||
|
|
||||||
|
### The final objective, and what it needs
|
||||||
|
|
||||||
|
> "that + an LLM way of diving in without being forced to rethink the code"
|
||||||
|
|
||||||
|
Two halves, and they are complementary. The **rung** says *this was decided*; the second half says the
|
||||||
|
developer must not have to re-derive *why*. The material for that already exists and is called the
|
||||||
|
**witness**: `username -> db.session.execute(text(user_query)) in models/user_model.py (line 73, 8 steps)` is
|
||||||
|
the reasoning, already written down, already attached to the finding.
|
||||||
|
|
||||||
|
So the objective is largely a plumbing question, and the session's recurring defect shape applies to it.
|
||||||
|
Checked rather than assumed: `openultrasast.get_finding` returns the whole record including the witness, but
|
||||||
|
`openultrasast.findings` was dropping the rung -- so an LLM listing libpng's 198 rows could not tell the one
|
||||||
|
the graph decided from the 197 an enumerator proposed without fetching all 198 individually. Fixed: the list
|
||||||
|
carries the rung, leads with what was decided, and returns a per-rung tally.
|
||||||
|
|
||||||
|
What still stands between here and the objective:
|
||||||
|
|
||||||
|
* **The witness is one line.** It names the source, the sink, the file and the step count, but not the steps.
|
||||||
|
For "dive in without rethinking", the path elements are the thing worth having, and the query already walks
|
||||||
|
them (`flow.elements`) before discarding all but the first.
|
||||||
|
* **The suspicion band drowns the signal** (2.11). An LLM given 163 "this API is present" rows will reason
|
||||||
|
about them, at length, and that is worse than a human ignoring them.
|
||||||
|
* **A family that cannot decide must say so** (2.16), or the LLM will confidently explain a silence.
|
||||||
|
|||||||
@@ -91,6 +91,13 @@ def _status(args: dict[str, Any]) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Strongest first. A caller listing a real repository's findings gets hundreds of rows, and without the
|
||||||
|
# rung it cannot tell the one the graph decided from the ones an enumerator proposed -- libpng returned 198
|
||||||
|
# rows of which one was entailed. It would have to fetch each finding individually to find out, which is the
|
||||||
|
# re-derivation this list exists to spare it.
|
||||||
|
_RUNG_ORDER = {"execution_confirmed": 0, "model_entailed": 1, "model_corroborated": 2, "suspicion": 3}
|
||||||
|
|
||||||
|
|
||||||
def _findings(args: dict[str, Any]) -> dict[str, Any]:
|
def _findings(args: dict[str, Any]) -> dict[str, Any]:
|
||||||
payload = _read_json(Path(_require(args, "run_dir")) / "findings.json")
|
payload = _read_json(Path(_require(args, "run_dir")) / "findings.json")
|
||||||
items = [
|
items = [
|
||||||
@@ -100,10 +107,15 @@ def _findings(args: dict[str, Any]) -> dict[str, Any]:
|
|||||||
"line": finding.get("line"),
|
"line": finding.get("line"),
|
||||||
"severity": finding["severity"],
|
"severity": finding["severity"],
|
||||||
"title": finding["title"],
|
"title": finding["title"],
|
||||||
|
"rung": finding.get("rung", "suspicion"),
|
||||||
}
|
}
|
||||||
for finding in payload.get("findings", [])
|
for finding in payload.get("findings", [])
|
||||||
]
|
]
|
||||||
return {"findings": items, "count": len(items)}
|
items.sort(key=lambda item: (_RUNG_ORDER.get(str(item["rung"]), len(_RUNG_ORDER)), str(item["path"]), item["line"] or 0))
|
||||||
|
by_rung: dict[str, int] = {}
|
||||||
|
for item in items:
|
||||||
|
by_rung[str(item["rung"])] = by_rung.get(str(item["rung"]), 0) + 1
|
||||||
|
return {"findings": items, "count": len(items), "by_rung": by_rung}
|
||||||
|
|
||||||
|
|
||||||
def _load_finding(run_dir: Path, finding_id: str) -> dict[str, Any]:
|
def _load_finding(run_dir: Path, finding_id: str) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -146,3 +146,30 @@ def test_serve_stdio_loop_handles_tools_list() -> None:
|
|||||||
assert serve(stdin, stdout) == 0
|
assert serve(stdin, stdout) == 0
|
||||||
response = json.loads(stdout.getvalue().strip())
|
response = json.loads(stdout.getvalue().strip())
|
||||||
assert {tool["name"] for tool in response["result"]["tools"]} == EXPECTED_TOOLS
|
assert {tool["name"] for tool in response["result"]["tools"]} == EXPECTED_TOOLS
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_findings_list_carries_the_rung_and_leads_with_what_was_decided(tmp_path: Path) -> None:
|
||||||
|
"""The final objective is an LLM diving in without re-deriving the reasoning.
|
||||||
|
|
||||||
|
Without the rung a caller cannot tell the finding the graph DECIDED from the ones an enumerator merely
|
||||||
|
proposed: libpng returned 198 rows of which one was entailed, and finding it would have meant fetching
|
||||||
|
all 198 individually. The evidence itself stays on `get_finding`; what the list owes is triage.
|
||||||
|
"""
|
||||||
|
run = tmp_path / "run"
|
||||||
|
run.mkdir()
|
||||||
|
(run / "findings.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"findings": [
|
||||||
|
{"finding_id": "b", "path": "z.c", "line": 9, "severity": "low", "title": "noise", "rung": "suspicion"},
|
||||||
|
{"finding_id": "a", "path": "a.py", "line": 73, "severity": "high", "title": "sqli", "rung": "model_entailed"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
result = _call(McpServer(), "openultrasast.findings", {"run_dir": str(run)})
|
||||||
|
|
||||||
|
assert [item["finding_id"] for item in result["findings"]] == ["a", "b"], "decided first"
|
||||||
|
assert result["findings"][0]["rung"] == "model_entailed"
|
||||||
|
assert result["by_rung"] == {"model_entailed": 1, "suspicion": 1}
|
||||||
|
|||||||
Reference in New Issue
Block a user