mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
f1c341df6c
5.1 inverts what the corpus is for. Previously a pair the detector missed was a
DETECTOR failure, to be averaged away over K runs and chased with a prompt
change. The arbiter is deterministic: it will miss that pair the same way every
time, and the only thing that moves it is better modelling. So a pair the model
cannot split is a NAMED GAP IN THE MODEL, recorded with its reason, and every
gap carries one even when the caller supplies none -- a bare gap teaches
nothing.
`covered` means the model reached a strictly higher rung on the vulnerable side
than on its fix. Flagging both sides equally is not coverage: it distinguishes
nothing, which is exactly what group 2 measured taint reachability doing on
every parameterised SQL pair. The harvest seam takes the fix commit and derives
the vulnerable side as its parent, with the fix diff as the oracle -- nothing
hand-labelled.
5.2 keeps three disciplines from the deleted publish layer, each because its
absence caused a real error here:
* per-slice rows always -- `render` RAISES without them. A mixed headline
once hid agent-written code scoring 44% against the development slice's 83%.
* the overfitting gap beside the headline, and ABSENT rather than zero when a
slice is missing: reading an unmeasured slice as "no overfitting" is the
flattering failure, not a neutral one.
* the rung on every finding, so a suspicion cannot be read as a result.
Run end to end against the committed ceiling artifact, the report says things
worth not hiding: agent-vfc 0/28 covered, vibe-py 40% against vfc-js 24%, an
overfitting gap of +16%, and gaps grouped by family with their reasons --
injection 32 (21 no verdict, 11 distinguished nothing), path 8 (all no
verdict), untrusted_destination 5 (4 entailed on BOTH sides).
713 passing, mypy/ruff clean, gates byte-identical, zero orphaned modules.
A full re-measure with the closure-scoping fix is in flight; these numbers are
the pre-fix baseline and will move.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
94 lines
4.3 KiB
Python
94 lines
4.3 KiB
Python
"""model-grounded-detection task 5.1: the corpus calibrates the MODEL, not an LLM vote (Req 9).
|
|
|
|
The inversion this feature rests on. Previously a pair the detector missed was a detector failure to be
|
|
averaged away over K runs. Here a pair the model cannot split is a *named gap in the model*, with the reason
|
|
recorded, because the model is deterministic: it will miss that pair the same way every time, and the only
|
|
thing that changes it is better modelling.
|
|
|
|
`covered` therefore means the model reached a strictly higher rung on the vulnerable side than on its fix --
|
|
not that it flagged something. Flagging both sides equally distinguishes nothing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
def _outcome(name, family="injection", slice_name="vibe-py", vuln="model_entailed", fixed="none", reason=""): # type: ignore[no-untyped-def]
|
|
from openultrasast.model.calibrate import PairOutcome
|
|
|
|
return PairOutcome(pair=name, family=family, slice=slice_name, vuln_rung=vuln, fixed_rung=fixed, reason=reason)
|
|
|
|
|
|
def test_a_pair_the_model_splits_is_covered() -> None:
|
|
from openultrasast.model.calibrate import calibrate
|
|
|
|
report = calibrate([_outcome("a")])
|
|
assert report.covered == ("a",)
|
|
assert report.gaps == ()
|
|
|
|
|
|
def test_a_pair_the_model_cannot_split_is_a_named_gap_not_a_miss() -> None:
|
|
"""Req 9.2: the reason travels with the gap, because a deterministic model fails the same way every time."""
|
|
from openultrasast.model.calibrate import calibrate
|
|
|
|
report = calibrate([_outcome("b", vuln="none", fixed="none", reason="sink_not_in_table")])
|
|
assert report.gaps and report.gaps[0].pair == "b"
|
|
assert report.gaps[0].reason == "sink_not_in_table"
|
|
assert report.covered == ()
|
|
|
|
|
|
def test_flagging_both_sides_equally_is_a_gap_not_a_success() -> None:
|
|
"""The model saw something on the fix too, so it distinguished nothing — that is the whole metric."""
|
|
from openultrasast.model.calibrate import calibrate
|
|
|
|
report = calibrate([_outcome("c", vuln="model_entailed", fixed="model_entailed")])
|
|
assert report.covered == ()
|
|
assert report.gaps[0].reason, "an unsplit pair must carry a reason even when none was supplied"
|
|
|
|
|
|
def test_gaps_are_grouped_by_family_with_their_reasons() -> None:
|
|
from openultrasast.model.calibrate import calibrate
|
|
|
|
report = calibrate([
|
|
_outcome("a", family="path", vuln="none", reason="sink_not_in_table"),
|
|
_outcome("b", family="path", vuln="none", reason="sink_not_in_table"),
|
|
_outcome("c", family="injection", vuln="none", reason="source_unmodelled"),
|
|
_outcome("d", family="injection"),
|
|
])
|
|
by_family = report.gaps_by_family()
|
|
assert set(by_family) == {"path", "injection"}
|
|
assert by_family["path"]["pairs"] == 2
|
|
assert by_family["path"]["reasons"] == {"sink_not_in_table": 2}
|
|
assert by_family["injection"]["pairs"] == 1
|
|
|
|
|
|
def test_the_report_is_per_slice_and_names_the_deciding_one() -> None:
|
|
"""Req 9.3/11.1: the real-world slice decides; a development slice must not stand in for the corpus."""
|
|
from openultrasast.model.calibrate import calibrate
|
|
|
|
report = calibrate([
|
|
_outcome("a", slice_name="vibe-py"),
|
|
_outcome("b", slice_name="vfc-js", vuln="none", reason="sink_not_in_table"),
|
|
])
|
|
assert set(report.per_slice) == {"vibe-py", "vfc-js"}
|
|
assert report.per_slice["vibe-py"]["covered"] == 1
|
|
assert report.per_slice["vfc-js"]["gaps"] == 1
|
|
|
|
|
|
def test_the_report_refuses_a_bare_aggregate() -> None:
|
|
from openultrasast.model.calibrate import calibrate
|
|
|
|
text = calibrate([_outcome("a"), _outcome("b", slice_name="vfc-js", vuln="none")]).to_markdown()
|
|
assert "vibe-py" in text and "vfc-js" in text, "no headline without its per-slice rows"
|
|
assert "coverage" in text.lower()
|
|
|
|
|
|
def test_a_cve_recipe_yields_the_parent_and_fix_commits() -> None:
|
|
"""Req 9.4: the harvest seam. The fix diff is the oracle; nothing is hand-labelled."""
|
|
from openultrasast.model.calibrate import harvest_recipe
|
|
|
|
recipe = harvest_recipe({"repo": "org/proj", "fix_commit": "abc123", "relpath": "app.py", "function": "run"})
|
|
assert recipe.repo == "org/proj"
|
|
assert recipe.fix_commit == "abc123"
|
|
assert recipe.parent_ref == "abc123^", "the vulnerable side is the fix commit's parent, by construction"
|
|
assert recipe.relpath == "app.py" and recipe.function == "run"
|