mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
fe45081b43
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.
Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.
Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 90.0%
leak 12.3% 6.7%
detection -- 96.7%
cost ~$0.69 $0.0068
27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).
One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.
The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
91 lines
3.6 KiB
Python
91 lines
3.6 KiB
Python
"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1).
|
|
|
|
Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a
|
|
real error in this project's history:
|
|
|
|
per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the
|
|
hand-written development slice scored 83%
|
|
the overfitting gap published beside the headline, so the distance between the development slice and the
|
|
real-world one is never something a reader has to compute
|
|
the rung on every a finding's strength is what established it; a report that omits the rung invites the
|
|
finding reader to treat a suspicion as a result
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
|
|
def _slice(name, pairs, covered): # type: ignore[no-untyped-def]
|
|
from openultrasast.model.report import SliceRow
|
|
|
|
return SliceRow(slice=name, pairs=pairs, covered=covered)
|
|
|
|
|
|
def test_the_report_carries_a_row_per_slice() -> None:
|
|
from openultrasast.model.report import render
|
|
|
|
text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)])
|
|
assert "vibe-py" in text and "vfc-js" in text
|
|
assert "30" in text and "17" in text
|
|
|
|
|
|
def test_a_bare_aggregate_is_refused() -> None:
|
|
"""Req 11.1: no cross-slice number without the rows it was computed from."""
|
|
from openultrasast.model.report import render
|
|
|
|
with pytest.raises(ValueError, match="per-slice"):
|
|
render([])
|
|
|
|
|
|
def test_the_overfitting_gap_is_published_beside_the_headline() -> None:
|
|
"""The development slice minus the deciding real-world slice, stated rather than left to be computed."""
|
|
from openultrasast.model.report import overfitting_gap, render
|
|
|
|
rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)]
|
|
gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js")
|
|
assert gap == pytest.approx(0.80 - 0.20)
|
|
assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower()
|
|
assert "+60" in render(rows, development="vibe-py", deciding="vfc-js")
|
|
|
|
|
|
def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None:
|
|
"""A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one."""
|
|
from openultrasast.model.report import overfitting_gap
|
|
|
|
assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None
|
|
|
|
|
|
def test_every_finding_shows_its_rung() -> None:
|
|
from openultrasast.findings import StaticFinding
|
|
from openultrasast.model.ladder import Rung
|
|
from openultrasast.model.report import render_findings
|
|
|
|
finding = StaticFinding(
|
|
finding_id="r:app.py:1",
|
|
path="app.py",
|
|
title="t",
|
|
severity="high",
|
|
confidence="medium",
|
|
evidence_level="static_corroboration",
|
|
rationale="why",
|
|
line=1,
|
|
function_name="run",
|
|
reachability_status="unknown",
|
|
reachability_evidence=[],
|
|
reachability_conditions=[],
|
|
tags=[],
|
|
ranking_priority=1.0,
|
|
rung=Rung.ENTAILED,
|
|
)
|
|
text = render_findings([finding])
|
|
assert "model_entailed" in text and "app.py" in text
|
|
|
|
|
|
def test_the_deciding_slice_is_named_in_the_report() -> None:
|
|
"""Req 9.3: the real-world slice decides, and the report says which one that is."""
|
|
from openultrasast.model.report import render
|
|
|
|
text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js")
|
|
assert "vfc-js" in text and "decid" in text.lower()
|