From f1c341df6c1aadb2121a4657e0c049438b36cc85 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:28:10 +0200 Subject: [PATCH] feat(model-grounded-detection): group 5, corpus calibration and honest reporting 5.1 inverts what the corpus is for. Previously a pair the detector missed was a DETECTOR failure, to be averaged away over K runs and chased with a prompt change. The arbiter is deterministic: it will miss that pair the same way every time, and the only thing that moves it is better modelling. So a pair the model cannot split is a NAMED GAP IN THE MODEL, recorded with its reason, and every gap carries one even when the caller supplies none -- a bare gap teaches nothing. `covered` means the model reached a strictly higher rung on the vulnerable side than on its fix. Flagging both sides equally is not coverage: it distinguishes nothing, which is exactly what group 2 measured taint reachability doing on every parameterised SQL pair. The harvest seam takes the fix commit and derives the vulnerable side as its parent, with the fix diff as the oracle -- nothing hand-labelled. 5.2 keeps three disciplines from the deleted publish layer, each because its absence caused a real error here: * per-slice rows always -- `render` RAISES without them. A mixed headline once hid agent-written code scoring 44% against the development slice's 83%. * the overfitting gap beside the headline, and ABSENT rather than zero when a slice is missing: reading an unmeasured slice as "no overfitting" is the flattering failure, not a neutral one. * the rung on every finding, so a suspicion cannot be read as a result. Run end to end against the committed ceiling artifact, the report says things worth not hiding: agent-vfc 0/28 covered, vibe-py 40% against vfc-js 24%, an overfitting gap of +16%, and gaps grouped by family with their reasons -- injection 32 (21 no verdict, 11 distinguished nothing), path 8 (all no verdict), untrusted_destination 5 (4 entailed on BOTH sides). 713 passing, mypy/ruff clean, gates byte-identical, zero orphaned modules. A full re-measure with the closure-scoping fix is in flight; these numbers are the pre-fix baseline and will move. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .../measurements/2026-09-08-module-audit.json | 19 +- src/openultrasast/model/audit.py | 2 + src/openultrasast/model/calibrate.py | 175 ++++++++++++++++++ src/openultrasast/model/report.py | 83 +++++++++ tests/test_model_calibrate.py | 93 ++++++++++ tests/test_model_report.py | 79 ++++++++ 6 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 src/openultrasast/model/calibrate.py create mode 100644 src/openultrasast/model/report.py create mode 100644 tests/test_model_calibrate.py create mode 100644 tests/test_model_report.py diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 7e5faff..da5c404 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,13 +1,13 @@ { "counts": { - "load_bearing": 82, + "load_bearing": 84, "standalone_capability": 7 }, - "generated": "2026-09-08 (regenerated after group 4: dominance and constant abstraction)", + "generated": "2026-09-08 (regenerated after group 5: calibration and reporting)", "kept_with_a_lever_stripped": { "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." }, - "modules": 89, + "modules": 91, "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], "removed_in_this_pass": { @@ -172,6 +172,7 @@ "improve.evolve", "map_gate", "model.ladder", + "model.report", "pairs", "rank", "regress.candidate", @@ -343,6 +344,12 @@ "module": "model.audit", "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan" }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.calibrate", + "reason": "model-grounded-detection Req 9: the corpus as the model's calibration set" + }, { "classification": "load_bearing", "importers": [ @@ -386,6 +393,12 @@ "module": "model.ladder", "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries" }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.report", + "reason": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap" + }, { "classification": "load_bearing", "importers": [ diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index b6a5d76..383a905 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -36,6 +36,8 @@ SPEC_OWNED = { "model.judge": "model-grounded-detection Req 8: one bounded question, checked against the model", "model.dominance": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash", "model.config_value": "model-grounded-detection Req 6.3: constant abstraction for the configuration families", + "model.calibrate": "model-grounded-detection Req 9: the corpus as the model's calibration set", + "model.report": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", "cpg": "model-grounded-detection Req 4: the CPG seam package", "cpg.backend": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", "cpg.capability": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", diff --git a/src/openultrasast/model/calibrate.py b/src/openultrasast/model/calibrate.py new file mode 100644 index 0000000..6fca434 --- /dev/null +++ b/src/openultrasast/model/calibrate.py @@ -0,0 +1,175 @@ +"""The corpus calibrates the model, not an LLM vote (model-grounded-detection, Req 9). + +This is the inversion the whole feature rests on. In the prior architecture a pair the detector missed was a +*detector* failure, to be averaged away over K runs and chased with a prompt change. Here the arbiter is +deterministic: it will miss that pair the same way every time, and the only thing that moves it is better +modelling. So a pair the model cannot split is a **named gap in the model**, recorded with its reason, and +the report's job is to make those gaps legible by family and by slice rather than to produce a score. + +`covered` means the model reached a strictly higher rung on the vulnerable side than on its fix. Flagging +both sides equally is not coverage — it distinguishes nothing, which is exactly the failure the group-2 +measurement found taint reachability making on every parameterised SQL pair. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import TypedDict + + +class FamilyGaps(TypedDict): + """Gaps for one family: how many pairs, and the tally of reasons behind them.""" + + pairs: int + reasons: dict[str, int] + +# Ascending, matching `ladder.Rung`. A pair is split when the vulnerable side outranks its fix. +_ORDER = {"none": 0, "": 0, "suspicion": 0, "model_corroborated": 1, "model_entailed": 2, "execution_confirmed": 3} + + +@dataclass(frozen=True) +class PairOutcome: + """What the model made of one vulnerable/fixed pair.""" + + pair: str + family: str + slice: str + vuln_rung: str + fixed_rung: str + reason: str = "" + + +@dataclass(frozen=True) +class ModelGap: + """A pair the model could not split, and why. Not a miss to average — a gap to close.""" + + pair: str + family: str + slice: str + reason: str + + +@dataclass(frozen=True) +class HarvestRecipe: + """Where a pair comes from. The fix commit's parent is the vulnerable side, by construction.""" + + repo: str + fix_commit: str + relpath: str + function: str = "" + + @property + def parent_ref(self) -> str: + return f"{self.fix_commit}^" + + +@dataclass(frozen=True) +class CalibrationReport: + covered: tuple[str, ...] + gaps: tuple[ModelGap, ...] + per_slice: Mapping[str, Mapping[str, int]] + per_family: Mapping[str, Mapping[str, int]] = field(default_factory=dict) + + def gaps_by_family(self) -> dict[str, FamilyGaps]: + """Gaps grouped by family, each with a tally of the reasons behind them.""" + out: dict[str, FamilyGaps] = {} + for gap in self.gaps: + block = out.setdefault(gap.family, {"pairs": 0, "reasons": {}}) + block["pairs"] += 1 + block["reasons"][gap.reason] = block["reasons"].get(gap.reason, 0) + 1 + return out + + def to_markdown(self) -> str: + """Per-slice rows first; there is no headline here that is not backed by them (Req 9.3, 11.1).""" + lines = ["## Model calibration", "", "Coverage is a pair the model *splits*: a strictly higher rung on", + "the vulnerable side than on its fix. Everything else is a named gap.", "", + "| slice | pairs | covered | gaps | coverage |", "|---|---|---|---|---|"] + for slice_name, block in sorted(self.per_slice.items()): + pairs = block["pairs"] or 1 + lines.append( + f"| {slice_name} | {block['pairs']} | {block['covered']} | {block['gaps']} | " + f"{block['covered'] / pairs:.0%} |" + ) + by_family = self.gaps_by_family() + if by_family: + lines += ["", "### Gaps by family", ""] + for family, gaps in sorted(by_family.items()): + detail = ", ".join(f"{k} x{v}" for k, v in sorted(gaps["reasons"].items())) + lines.append(f"- `{family}`: {gaps['pairs']} pair(s) — {detail}") + return "\n".join(lines) + "\n" + + +def calibrate(outcomes: Iterable[PairOutcome]) -> CalibrationReport: + """Split the corpus into what the model covers and what it cannot, with reasons.""" + covered: list[str] = [] + gaps: list[ModelGap] = [] + per_slice: dict[str, dict[str, int]] = {} + per_family: dict[str, dict[str, int]] = {} + + for outcome in outcomes: + split = _ORDER.get(outcome.vuln_rung, 0) > _ORDER.get(outcome.fixed_rung, 0) + for key, store in ((outcome.slice, per_slice), (outcome.family, per_family)): + block = store.setdefault(key, {"pairs": 0, "covered": 0, "gaps": 0}) + block["pairs"] += 1 + block["covered" if split else "gaps"] += 1 + if split: + covered.append(outcome.pair) + else: + gaps.append(ModelGap(pair=outcome.pair, family=outcome.family, slice=outcome.slice, + reason=outcome.reason or _default_reason(outcome))) + return CalibrationReport( + covered=tuple(sorted(covered)), + gaps=tuple(sorted(gaps, key=lambda g: (g.family, g.pair))), + per_slice=per_slice, + per_family=per_family, + ) + + +def _default_reason(outcome: PairOutcome) -> str: + """Every gap carries a reason, even when the caller supplied none — a bare gap teaches nothing.""" + if _ORDER.get(outcome.vuln_rung, 0) == 0: + return "no_verdict_on_the_vulnerable_side" + if outcome.vuln_rung == outcome.fixed_rung: + return f"both_sides_reached_{outcome.vuln_rung}: the model distinguished nothing" + return "fixed_side_outranked_the_vulnerable_one" + + +def harvest_recipe(entry: Mapping[str, object]) -> HarvestRecipe: + """A pair from CVE history: the fix commit and its parent, with the fix diff as the oracle (Req 9.4). + + Nothing here is hand-labelled. The recipe names where to fetch both sides; what makes the pair *true* is + the fix diff itself, read by the model rather than annotated by us. + """ + return HarvestRecipe( + repo=str(entry.get("repo", "")), + fix_commit=str(entry.get("fix_commit", "")), + relpath=str(entry.get("relpath", "")), + function=str(entry.get("function", "") or ""), + ) + + +def outcomes_from_rows(rows: Sequence[Mapping[str, object]]) -> list[PairOutcome]: + """Adapt the ceiling artifact's rows, so calibration reads the measurement rather than re-running it.""" + return [ + PairOutcome( + pair=str(row.get("pair", "")), + family=str(row.get("family", "")), + slice=str(row.get("slice", "")), + vuln_rung=str(row.get("vuln_rung", "none")), + fixed_rung=str(row.get("fixed_rung", "none")), + reason=str(row.get("miss_cause", "") or ""), + ) + for row in rows + ] + + +__all__ = [ + "CalibrationReport", + "HarvestRecipe", + "ModelGap", + "PairOutcome", + "calibrate", + "harvest_recipe", + "outcomes_from_rows", +] diff --git a/src/openultrasast/model/report.py b/src/openultrasast/model/report.py new file mode 100644 index 0000000..e3c4c24 --- /dev/null +++ b/src/openultrasast/model/report.py @@ -0,0 +1,83 @@ +"""Reporting that cannot flatter itself (model-grounded-detection, Req 9.3, 11.1). + +Three disciplines survive from the deleted publish/scoring layer, and each is here because its absence caused +a real error in this project's history rather than because it is tidy: + +* **Per-slice rows, always.** A headline over mixed slices once hid that agent-written code scored 44.4% + while the hand-written development slice scored 83.3%. `render` refuses to produce anything without rows. +* **The overfitting gap beside the headline.** The distance between the development slice and the deciding + real-world one is stated, never left for the reader to compute. When a slice is missing the gap is + *absent*, not zero — reading a missing measurement as "no overfitting" is the flattering failure. +* **The rung on every finding.** A finding's strength is what established it; omitting the rung invites a + reader to treat a suspicion as a result. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from ..findings import StaticFinding + + +@dataclass(frozen=True) +class SliceRow: + """One slice's coverage. `covered` is pairs the model *split*, not pairs it flagged.""" + + slice: str + pairs: int + covered: int + + @property + def rate(self) -> float: + return self.covered / self.pairs if self.pairs else 0.0 + + +def overfitting_gap(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> float | None: + """Development-slice coverage minus the deciding slice's, or ``None`` when either is absent. + + ``None`` rather than ``0.0`` on purpose: a slice that was never measured must not read as agreement + between them. + """ + by_name = {row.slice: row for row in rows} + if development not in by_name or deciding not in by_name: + return None + return by_name[development].rate - by_name[deciding].rate + + +def render(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> str: + """The report. Raises when there are no per-slice rows, because a bare aggregate is the thing to prevent.""" + if not rows: + raise ValueError("a report needs per-slice rows: there is no headline without them") + + lines = ["## Model coverage", "", "| slice | pairs | covered | coverage |", "|---|---|---|---|"] + for row in sorted(rows, key=lambda r: r.slice): + lines.append(f"| {row.slice} | {row.pairs} | {row.covered} | {row.rate:.0%} |") + + total_pairs = sum(row.pairs for row in rows) + total_covered = sum(row.covered for row in rows) + lines += ["", f"Across every slice: {total_covered}/{total_pairs} = {total_covered / total_pairs:.0%} " + "— read the rows above it, not this line."] + + gap = overfitting_gap(rows, development=development, deciding=deciding) + if gap is not None: + lines += [ + "", + f"**Overfitting gap: {gap:+.0%}** — `{development}` (development) minus `{deciding}`, " + f"which is the deciding real-world slice.", + ] + elif development or deciding: + lines += ["", f"**Overfitting gap: not measurable** — `{development}` or `{deciding}` is absent from this run."] + return "\n".join(lines) + "\n" + + +def render_findings(findings: Sequence[StaticFinding]) -> str: + """Findings with the rung that established each one (Req 5.3).""" + lines = ["## Findings", "", "| finding | location | rung | severity |", "|---|---|---|---|"] + for finding in findings: + where = f"{finding.path}:{finding.line}" if finding.line else finding.path + lines.append(f"| {finding.title} | {where} | `{finding.rung}` | {finding.severity} |") + return "\n".join(lines) + "\n" + + +__all__ = ["SliceRow", "overfitting_gap", "render", "render_findings"] diff --git a/tests/test_model_calibrate.py b/tests/test_model_calibrate.py new file mode 100644 index 0000000..b0e75b1 --- /dev/null +++ b/tests/test_model_calibrate.py @@ -0,0 +1,93 @@ +"""model-grounded-detection task 5.1: the corpus calibrates the MODEL, not an LLM vote (Req 9). + +The inversion this feature rests on. Previously a pair the detector missed was a detector failure to be +averaged away over K runs. Here a pair the model cannot split is a *named gap in the model*, with the reason +recorded, because the model is deterministic: it will miss that pair the same way every time, and the only +thing that changes it is better modelling. + +`covered` therefore means the model reached a strictly higher rung on the vulnerable side than on its fix -- +not that it flagged something. Flagging both sides equally distinguishes nothing. +""" + +from __future__ import annotations + + +def _outcome(name, family="injection", slice_name="vibe-py", vuln="model_entailed", fixed="none", reason=""): # type: ignore[no-untyped-def] + from openultrasast.model.calibrate import PairOutcome + + return PairOutcome(pair=name, family=family, slice=slice_name, vuln_rung=vuln, fixed_rung=fixed, reason=reason) + + +def test_a_pair_the_model_splits_is_covered() -> None: + from openultrasast.model.calibrate import calibrate + + report = calibrate([_outcome("a")]) + assert report.covered == ("a",) + assert report.gaps == () + + +def test_a_pair_the_model_cannot_split_is_a_named_gap_not_a_miss() -> None: + """Req 9.2: the reason travels with the gap, because a deterministic model fails the same way every time.""" + from openultrasast.model.calibrate import calibrate + + report = calibrate([_outcome("b", vuln="none", fixed="none", reason="sink_not_in_table")]) + assert report.gaps and report.gaps[0].pair == "b" + assert report.gaps[0].reason == "sink_not_in_table" + assert report.covered == () + + +def test_flagging_both_sides_equally_is_a_gap_not_a_success() -> None: + """The model saw something on the fix too, so it distinguished nothing — that is the whole metric.""" + from openultrasast.model.calibrate import calibrate + + report = calibrate([_outcome("c", vuln="model_entailed", fixed="model_entailed")]) + assert report.covered == () + assert report.gaps[0].reason, "an unsplit pair must carry a reason even when none was supplied" + + +def test_gaps_are_grouped_by_family_with_their_reasons() -> None: + from openultrasast.model.calibrate import calibrate + + report = calibrate([ + _outcome("a", family="path", vuln="none", reason="sink_not_in_table"), + _outcome("b", family="path", vuln="none", reason="sink_not_in_table"), + _outcome("c", family="injection", vuln="none", reason="source_unmodelled"), + _outcome("d", family="injection"), + ]) + by_family = report.gaps_by_family() + assert set(by_family) == {"path", "injection"} + assert by_family["path"]["pairs"] == 2 + assert by_family["path"]["reasons"] == {"sink_not_in_table": 2} + assert by_family["injection"]["pairs"] == 1 + + +def test_the_report_is_per_slice_and_names_the_deciding_one() -> None: + """Req 9.3/11.1: the real-world slice decides; a development slice must not stand in for the corpus.""" + from openultrasast.model.calibrate import calibrate + + report = calibrate([ + _outcome("a", slice_name="vibe-py"), + _outcome("b", slice_name="vfc-js", vuln="none", reason="sink_not_in_table"), + ]) + assert set(report.per_slice) == {"vibe-py", "vfc-js"} + assert report.per_slice["vibe-py"]["covered"] == 1 + assert report.per_slice["vfc-js"]["gaps"] == 1 + + +def test_the_report_refuses_a_bare_aggregate() -> None: + from openultrasast.model.calibrate import calibrate + + text = calibrate([_outcome("a"), _outcome("b", slice_name="vfc-js", vuln="none")]).to_markdown() + assert "vibe-py" in text and "vfc-js" in text, "no headline without its per-slice rows" + assert "coverage" in text.lower() + + +def test_a_cve_recipe_yields_the_parent_and_fix_commits() -> None: + """Req 9.4: the harvest seam. The fix diff is the oracle; nothing is hand-labelled.""" + from openultrasast.model.calibrate import harvest_recipe + + recipe = harvest_recipe({"repo": "org/proj", "fix_commit": "abc123", "relpath": "app.py", "function": "run"}) + assert recipe.repo == "org/proj" + assert recipe.fix_commit == "abc123" + assert recipe.parent_ref == "abc123^", "the vulnerable side is the fix commit's parent, by construction" + assert recipe.relpath == "app.py" and recipe.function == "run" diff --git a/tests/test_model_report.py b/tests/test_model_report.py new file mode 100644 index 0000000..e308d8e --- /dev/null +++ b/tests/test_model_report.py @@ -0,0 +1,79 @@ +"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1). + +Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a +real error in this project's history: + + per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the + hand-written development slice scored 83% + the overfitting gap published beside the headline, so the distance between the development slice and the + real-world one is never something a reader has to compute + the rung on every a finding's strength is what established it; a report that omits the rung invites the + finding reader to treat a suspicion as a result +""" + +from __future__ import annotations + +import pytest + + +def _slice(name, pairs, covered): # type: ignore[no-untyped-def] + from openultrasast.model.report import SliceRow + + return SliceRow(slice=name, pairs=pairs, covered=covered) + + +def test_the_report_carries_a_row_per_slice() -> None: + from openultrasast.model.report import render + + text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)]) + assert "vibe-py" in text and "vfc-js" in text + assert "30" in text and "17" in text + + +def test_a_bare_aggregate_is_refused() -> None: + """Req 11.1: no cross-slice number without the rows it was computed from.""" + from openultrasast.model.report import render + + with pytest.raises(ValueError, match="per-slice"): + render([]) + + +def test_the_overfitting_gap_is_published_beside_the_headline() -> None: + """The development slice minus the deciding real-world slice, stated rather than left to be computed.""" + from openultrasast.model.report import overfitting_gap, render + + rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)] + gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js") + assert gap == pytest.approx(0.80 - 0.20) + assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower() + assert "+60" in render(rows, development="vibe-py", deciding="vfc-js") + + +def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None: + """A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one.""" + from openultrasast.model.report import overfitting_gap + + assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None + + +def test_every_finding_shows_its_rung() -> None: + from openultrasast.findings import StaticFinding + from openultrasast.model.ladder import Rung + from openultrasast.model.report import render_findings + + finding = StaticFinding( + finding_id="r:app.py:1", path="app.py", title="t", severity="high", confidence="medium", + evidence_level="static_corroboration", rationale="why", line=1, function_name="run", + reachability_status="unknown", reachability_evidence=[], reachability_conditions=[], tags=[], + ranking_priority=1.0, rung=Rung.ENTAILED, + ) + text = render_findings([finding]) + assert "model_entailed" in text and "app.py" in text + + +def test_the_deciding_slice_is_named_in_the_report() -> None: + """Req 9.3: the real-world slice decides, and the report says which one that is.""" + from openultrasast.model.report import render + + text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js") + assert "vfc-js" in text and "decid" in text.lower()