mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
b8f8f727c6
3.1 adds StaticFinding.rung beside evidence_level rather than renaming it. The existing evidence_level carries `static_corroboration`, emitted by the flat IR overlay -- a path Req 4.4 demotes to the suspicion band. Restating that as model_corroborated would claim a CPG arbitrated something it never saw, and rewriting every legacy finding to suspicion would be a product-visible regression nobody asked for. So `rung` defaults to suspicion and is raised only by ladder.at_rung(finding, verdict), which is the structural form of Req 5.4: nothing rises above suspicion without a Verdict behind it. The witness travels with the rung, because a rung without its evidence is just a louder assertion. It reaches markdown, JSON and SARIF. 3.2 is the judge, and it is defined as much by what it lacks. Three exhaustive outcomes: the model entails (report it, never call the model), the model contradicts (drop the claim, record why), the model cannot decide (report suspicion). The LLM gets one bounded typed question with tools=[] so it cannot search for the site, asked only about the residual the graph cannot settle -- whether a sanitizer on a REAL flow suffices. Its answer can only confirm a flow the CPG already found; it can never conjure one. Req 8.5 is asserted structurally: a test parses judge.py and fails on any AST name like k_runs/majority/vote/average/acceptance/floor/budget, any for/while loop, or an import from the deleted round machinery. That test first matched the word "votes" in the module's own docstring, which is why it now inspects code rather than text. Verified live against a real CPG, not only scripted rows: vuln rung=model_entailed request.args["name"] -> db.execute(...) fixed rung=suspicion no source reaches this sink model calls made: 0 Both sides arbitrated by the graph with the LLM never consulted -- the thing the prior architecture could not do at any price. 678 passing, mypy/ruff clean, gates byte-identical, zero orphaned modules. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
96 lines
4.1 KiB
Python
96 lines
4.1 KiB
Python
"""model-grounded-detection task 3.1: the evidence ladder is the verdict type (Req 5).
|
|
|
|
The rung says what ESTABLISHED a finding, so a deterministic model result is never confused with an LLM
|
|
suspicion. The integrity constraint is Req 5.4: nothing reaches a rung above `suspicion` without a `Verdict`,
|
|
and a `Verdict` above `suspicion` is only ever produced by the CPG with no model call in the establishing step.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def _finding(**kw: object): # type: ignore[no-untyped-def]
|
|
from openultrasast.findings import StaticFinding
|
|
|
|
base = dict(
|
|
finding_id="r1:app.py:1", path="app.py", title="t", severity="high", confidence="medium",
|
|
evidence_level="static_corroboration", rationale="why", line=1, function_name="run",
|
|
reachability_status="unknown", reachability_evidence=[], reachability_conditions=[], tags=[],
|
|
ranking_priority=1.0,
|
|
)
|
|
base.update(kw)
|
|
return StaticFinding(**base) # type: ignore[arg-type]
|
|
|
|
|
|
def test_the_rungs_are_ordered_and_comparable() -> None:
|
|
from openultrasast.model.ladder import Rung, at_or_above
|
|
|
|
assert at_or_above(Rung.ENTAILED, Rung.CORROBORATED)
|
|
assert at_or_above(Rung.CORROBORATED, Rung.SUSPICION)
|
|
assert not at_or_above(Rung.SUSPICION, Rung.CORROBORATED)
|
|
assert at_or_above(Rung.EXECUTION_CONFIRMED, Rung.ENTAILED)
|
|
|
|
|
|
def test_a_finding_defaults_to_suspicion() -> None:
|
|
"""Req 5.4: a claim nothing has arbitrated is a suspicion, whatever produced it."""
|
|
from openultrasast.model.ladder import Rung
|
|
|
|
assert _finding().rung == Rung.SUSPICION
|
|
|
|
|
|
def test_only_a_verdict_raises_a_finding_above_suspicion() -> None:
|
|
from openultrasast.model.ladder import Rung, Verdict, at_rung
|
|
|
|
finding = _finding()
|
|
verdict = Verdict(rung=Rung.ENTAILED, family="injection", witness="src -> sink")
|
|
raised = at_rung(finding, verdict)
|
|
assert raised.rung == Rung.ENTAILED
|
|
assert "src -> sink" in raised.rationale, "the witness travels with the rung it justifies"
|
|
assert raised.finding_id == finding.finding_id and raised.line == finding.line
|
|
|
|
|
|
def test_a_none_verdict_leaves_the_finding_at_suspicion() -> None:
|
|
from openultrasast.model.ladder import Rung, at_rung
|
|
|
|
assert at_rung(_finding(), None).rung == Rung.SUSPICION
|
|
|
|
|
|
def test_the_legacy_evidence_level_is_left_alone() -> None:
|
|
"""`static_corroboration` is the *flat IR overlay's* word and Req 4.4 demotes that path to the suspicion
|
|
band. Restating it as `model_corroborated` would claim a CPG arbitrated something it never saw."""
|
|
from openultrasast.model.ladder import Rung, Verdict, at_rung
|
|
|
|
raised = at_rung(_finding(), Verdict(rung=Rung.CORROBORATED, family="injection", witness="w"))
|
|
assert raised.evidence_level == "static_corroboration", "the legacy field keeps its own meaning"
|
|
assert raised.rung == Rung.CORROBORATED
|
|
|
|
|
|
def test_the_rung_reaches_markdown_json_and_sarif(tmp_path: Path) -> None:
|
|
"""Req 5.3: every output format carries the rung."""
|
|
from openultrasast.findings import write_findings
|
|
from openultrasast.model.ladder import Rung, Verdict, at_rung
|
|
from openultrasast.reports import write_markdown_report, write_sarif_report
|
|
|
|
finding = at_rung(_finding(), Verdict(rung=Rung.ENTAILED, family="injection", witness="src -> sink"))
|
|
md, js, sarif = tmp_path / "r.md", tmp_path / "r.json", tmp_path / "r.sarif"
|
|
write_markdown_report([finding], md)
|
|
write_findings([finding], js)
|
|
write_sarif_report([finding], [], sarif)
|
|
|
|
assert "model_entailed" in md.read_text()
|
|
assert json.loads(js.read_text())["findings"][0]["rung"] == "model_entailed"
|
|
props = json.loads(sarif.read_text())["runs"][0]["results"][0]["properties"]
|
|
assert props["rung"] == "model_entailed"
|
|
|
|
|
|
def test_every_rung_round_trips_through_json(tmp_path: Path) -> None:
|
|
from openultrasast.findings import write_findings
|
|
from openultrasast.model.ladder import Rung
|
|
|
|
for rung in Rung:
|
|
out = tmp_path / f"{rung.value}.json"
|
|
write_findings([_finding(rung=rung)], out)
|
|
assert json.loads(out.read_text())["findings"][0]["rung"] == rung.value
|