mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
feat(reports): the model's own suspicion band is a proposal too
contributor-scan 2.17, completing Req 5.5. Task 2.11 made static pattern matches proportionate and left the model layer's own suspicions alone, reasoning that a proposer had reasoned about them. That does not survive inspection. A model suspicion is what the enumerator proposed at a site the graph could NOT decide, affirmed by a judge with no coverage there -- and its own rationale says exactly that: "the model resolved no flow here; its sink table may not cover this API". It is a pattern match with an opinion attached, and it reads like a reasoned claim only because a model produced it. vampi 60 findings, 61 sections -> 26, report 1085 lines -> 446 libpng 198 findings, 39 sections -> 30 **What it costs, which is the number that decides whether the rule is right.** For each known vulnerability the recipe declares, take every finding row at its location and ask whether any survives the rule. All three of vampi's -- the SQL injection and both documented BOLAs -- are still found at model_entailed and all three survive. Nothing known is hidden. Three vulnerabilities in one repository is also a statement about how small the evidence base is, and the artifact says so. The obligations checker sits at `suspicion` too and is NOT grouped: its output comes from a structural analysis, an obligated operation with no discharging guard, rather than from a guess. An earlier attempt at this rule filtered on rung alone and would have collapsed it; six tests failed and they were right to. "Pattern matches" is now "Proposed, not corroborated", since it holds two shapes. Model rows group by FAMILY rather than by rule, because their rule is always `model`. Gates byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
af7e33ef3e
commit
3e1d13e4b6
+67
-2
@@ -485,7 +485,7 @@ def test_a_pattern_rule_with_many_sites_is_summarised_not_repeated(tmp_path: Pat
|
||||
write_markdown_report(crowded, output)
|
||||
|
||||
text = output.read_text()
|
||||
assert "## Pattern matches" in text
|
||||
assert "## Proposed, not corroborated" in text
|
||||
assert "20 site(s) across 20 file(s)" in text
|
||||
assert text.count("## c-unsafe-copy needs review") == 0, "twenty sections for one rule is the noise"
|
||||
assert "`findings.json`" in text, "counted, never dropped"
|
||||
@@ -499,7 +499,7 @@ def test_a_pattern_rule_with_few_sites_keeps_its_sections(tmp_path: Path) -> Non
|
||||
|
||||
text = output.read_text()
|
||||
assert "## python-flask-debug needs review" in text
|
||||
assert "## Pattern matches" not in text
|
||||
assert "## Proposed, not corroborated" not in text
|
||||
|
||||
|
||||
def test_a_reasoned_claim_keeps_its_section_whatever_its_rung(tmp_path: Path) -> None:
|
||||
@@ -552,3 +552,68 @@ def test_a_scan_that_made_no_claims_states_no_limits(tmp_path: Path) -> None:
|
||||
write_markdown_report([_finding()], output)
|
||||
|
||||
assert "## What was analysed" not in output.read_text()
|
||||
|
||||
|
||||
def test_a_model_suspicion_is_grouped_like_a_pattern_match(tmp_path: Path) -> None:
|
||||
"""contributor-scan 2.17, Req 5.5's other half.
|
||||
|
||||
A model suspicion is what the enumerator proposed at a site the graph could NOT decide, affirmed by a
|
||||
judge with no coverage there -- a pattern match with an opinion attached. It reads like a reasoned claim
|
||||
because a model produced it, and on vampi it was 38 of 60 findings on a 520-line application.
|
||||
"""
|
||||
from dataclasses import replace
|
||||
|
||||
base = _finding()
|
||||
suspicions = [
|
||||
replace(
|
||||
base,
|
||||
finding_id=f"model:injection:api/f{i}.py:{i}:jsonify",
|
||||
evidence_level="suspicion",
|
||||
rung="suspicion",
|
||||
path=f"api/f{i}.py",
|
||||
line=i,
|
||||
rationale="the model resolved no flow here; its sink table may not cover this API",
|
||||
)
|
||||
for i in range(20)
|
||||
]
|
||||
output = tmp_path / "report.md"
|
||||
|
||||
write_markdown_report(suspicions, output)
|
||||
|
||||
text = output.read_text()
|
||||
assert "`model:injection`" in text, "grouped by family, since the rule is always `model`"
|
||||
assert "20 site(s)" in text
|
||||
assert text.count(f"## {base.title}") == 0
|
||||
|
||||
|
||||
def test_an_obligation_finding_is_not_grouped_away(tmp_path: Path) -> None:
|
||||
"""It sits at `suspicion` too, but it comes from a structural analysis rather than a guess. Collapsing
|
||||
it would hide a detector -- which an earlier attempt at this rule did, and six tests caught it."""
|
||||
from dataclasses import replace
|
||||
|
||||
base = _finding()
|
||||
obligations = [
|
||||
replace(base, finding_id=f"obligation:protected_read:api/u.py:{i}", evidence_level="suspicion", rung="suspicion", line=i)
|
||||
for i in range(20)
|
||||
]
|
||||
output = tmp_path / "report.md"
|
||||
|
||||
write_markdown_report(obligations, output)
|
||||
|
||||
assert output.read_text().count(f"## {base.title}") == 20
|
||||
|
||||
|
||||
def test_an_entailed_model_finding_keeps_its_section(tmp_path: Path) -> None:
|
||||
"""The rung is the product. Whatever the model layer DECIDED is never summarised away."""
|
||||
from dataclasses import replace
|
||||
|
||||
base = _finding()
|
||||
entailed = [
|
||||
replace(base, finding_id=f"model:injection:app.py:{i}:execute", evidence_level="suspicion", rung="model_entailed", line=i)
|
||||
for i in range(20)
|
||||
]
|
||||
output = tmp_path / "report.md"
|
||||
|
||||
write_markdown_report(entailed, output)
|
||||
|
||||
assert output.read_text().count(f"## {base.title}") == 20
|
||||
|
||||
Reference in New Issue
Block a user