diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 7e5faff..da5c404 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,13 +1,13 @@ { "counts": { - "load_bearing": 82, + "load_bearing": 84, "standalone_capability": 7 }, - "generated": "2026-09-08 (regenerated after group 4: dominance and constant abstraction)", + "generated": "2026-09-08 (regenerated after group 5: calibration and reporting)", "kept_with_a_lever_stripped": { "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." }, - "modules": 89, + "modules": 91, "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], "removed_in_this_pass": { @@ -172,6 +172,7 @@ "improve.evolve", "map_gate", "model.ladder", + "model.report", "pairs", "rank", "regress.candidate", @@ -343,6 +344,12 @@ "module": "model.audit", "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan" }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.calibrate", + "reason": "model-grounded-detection Req 9: the corpus as the model's calibration set" + }, { "classification": "load_bearing", "importers": [ @@ -386,6 +393,12 @@ "module": "model.ladder", "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries" }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.report", + "reason": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap" + }, { "classification": "load_bearing", "importers": [ diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index b6a5d76..383a905 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -36,6 +36,8 @@ SPEC_OWNED = { "model.judge": "model-grounded-detection Req 8: one bounded question, checked against the model", "model.dominance": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash", "model.config_value": "model-grounded-detection Req 6.3: constant abstraction for the configuration families", + "model.calibrate": "model-grounded-detection Req 9: the corpus as the model's calibration set", + "model.report": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", "cpg": "model-grounded-detection Req 4: the CPG seam package", "cpg.backend": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", "cpg.capability": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", diff --git a/src/openultrasast/model/calibrate.py b/src/openultrasast/model/calibrate.py new file mode 100644 index 0000000..6fca434 --- /dev/null +++ b/src/openultrasast/model/calibrate.py @@ -0,0 +1,175 @@ +"""The corpus calibrates the model, not an LLM vote (model-grounded-detection, Req 9). + +This is the inversion the whole feature rests on. In the prior architecture a pair the detector missed was a +*detector* failure, to be averaged away over K runs and chased with a prompt change. Here the arbiter is +deterministic: it will miss that pair the same way every time, and the only thing that moves it is better +modelling. So a pair the model cannot split is a **named gap in the model**, recorded with its reason, and +the report's job is to make those gaps legible by family and by slice rather than to produce a score. + +`covered` means the model reached a strictly higher rung on the vulnerable side than on its fix. Flagging +both sides equally is not coverage — it distinguishes nothing, which is exactly the failure the group-2 +measurement found taint reachability making on every parameterised SQL pair. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import TypedDict + + +class FamilyGaps(TypedDict): + """Gaps for one family: how many pairs, and the tally of reasons behind them.""" + + pairs: int + reasons: dict[str, int] + +# Ascending, matching `ladder.Rung`. A pair is split when the vulnerable side outranks its fix. +_ORDER = {"none": 0, "": 0, "suspicion": 0, "model_corroborated": 1, "model_entailed": 2, "execution_confirmed": 3} + + +@dataclass(frozen=True) +class PairOutcome: + """What the model made of one vulnerable/fixed pair.""" + + pair: str + family: str + slice: str + vuln_rung: str + fixed_rung: str + reason: str = "" + + +@dataclass(frozen=True) +class ModelGap: + """A pair the model could not split, and why. Not a miss to average — a gap to close.""" + + pair: str + family: str + slice: str + reason: str + + +@dataclass(frozen=True) +class HarvestRecipe: + """Where a pair comes from. The fix commit's parent is the vulnerable side, by construction.""" + + repo: str + fix_commit: str + relpath: str + function: str = "" + + @property + def parent_ref(self) -> str: + return f"{self.fix_commit}^" + + +@dataclass(frozen=True) +class CalibrationReport: + covered: tuple[str, ...] + gaps: tuple[ModelGap, ...] + per_slice: Mapping[str, Mapping[str, int]] + per_family: Mapping[str, Mapping[str, int]] = field(default_factory=dict) + + def gaps_by_family(self) -> dict[str, FamilyGaps]: + """Gaps grouped by family, each with a tally of the reasons behind them.""" + out: dict[str, FamilyGaps] = {} + for gap in self.gaps: + block = out.setdefault(gap.family, {"pairs": 0, "reasons": {}}) + block["pairs"] += 1 + block["reasons"][gap.reason] = block["reasons"].get(gap.reason, 0) + 1 + return out + + def to_markdown(self) -> str: + """Per-slice rows first; there is no headline here that is not backed by them (Req 9.3, 11.1).""" + lines = ["## Model calibration", "", "Coverage is a pair the model *splits*: a strictly higher rung on", + "the vulnerable side than on its fix. Everything else is a named gap.", "", + "| slice | pairs | covered | gaps | coverage |", "|---|---|---|---|---|"] + for slice_name, block in sorted(self.per_slice.items()): + pairs = block["pairs"] or 1 + lines.append( + f"| {slice_name} | {block['pairs']} | {block['covered']} | {block['gaps']} | " + f"{block['covered'] / pairs:.0%} |" + ) + by_family = self.gaps_by_family() + if by_family: + lines += ["", "### Gaps by family", ""] + for family, gaps in sorted(by_family.items()): + detail = ", ".join(f"{k} x{v}" for k, v in sorted(gaps["reasons"].items())) + lines.append(f"- `{family}`: {gaps['pairs']} pair(s) — {detail}") + return "\n".join(lines) + "\n" + + +def calibrate(outcomes: Iterable[PairOutcome]) -> CalibrationReport: + """Split the corpus into what the model covers and what it cannot, with reasons.""" + covered: list[str] = [] + gaps: list[ModelGap] = [] + per_slice: dict[str, dict[str, int]] = {} + per_family: dict[str, dict[str, int]] = {} + + for outcome in outcomes: + split = _ORDER.get(outcome.vuln_rung, 0) > _ORDER.get(outcome.fixed_rung, 0) + for key, store in ((outcome.slice, per_slice), (outcome.family, per_family)): + block = store.setdefault(key, {"pairs": 0, "covered": 0, "gaps": 0}) + block["pairs"] += 1 + block["covered" if split else "gaps"] += 1 + if split: + covered.append(outcome.pair) + else: + gaps.append(ModelGap(pair=outcome.pair, family=outcome.family, slice=outcome.slice, + reason=outcome.reason or _default_reason(outcome))) + return CalibrationReport( + covered=tuple(sorted(covered)), + gaps=tuple(sorted(gaps, key=lambda g: (g.family, g.pair))), + per_slice=per_slice, + per_family=per_family, + ) + + +def _default_reason(outcome: PairOutcome) -> str: + """Every gap carries a reason, even when the caller supplied none — a bare gap teaches nothing.""" + if _ORDER.get(outcome.vuln_rung, 0) == 0: + return "no_verdict_on_the_vulnerable_side" + if outcome.vuln_rung == outcome.fixed_rung: + return f"both_sides_reached_{outcome.vuln_rung}: the model distinguished nothing" + return "fixed_side_outranked_the_vulnerable_one" + + +def harvest_recipe(entry: Mapping[str, object]) -> HarvestRecipe: + """A pair from CVE history: the fix commit and its parent, with the fix diff as the oracle (Req 9.4). + + Nothing here is hand-labelled. The recipe names where to fetch both sides; what makes the pair *true* is + the fix diff itself, read by the model rather than annotated by us. + """ + return HarvestRecipe( + repo=str(entry.get("repo", "")), + fix_commit=str(entry.get("fix_commit", "")), + relpath=str(entry.get("relpath", "")), + function=str(entry.get("function", "") or ""), + ) + + +def outcomes_from_rows(rows: Sequence[Mapping[str, object]]) -> list[PairOutcome]: + """Adapt the ceiling artifact's rows, so calibration reads the measurement rather than re-running it.""" + return [ + PairOutcome( + pair=str(row.get("pair", "")), + family=str(row.get("family", "")), + slice=str(row.get("slice", "")), + vuln_rung=str(row.get("vuln_rung", "none")), + fixed_rung=str(row.get("fixed_rung", "none")), + reason=str(row.get("miss_cause", "") or ""), + ) + for row in rows + ] + + +__all__ = [ + "CalibrationReport", + "HarvestRecipe", + "ModelGap", + "PairOutcome", + "calibrate", + "harvest_recipe", + "outcomes_from_rows", +] diff --git a/src/openultrasast/model/report.py b/src/openultrasast/model/report.py new file mode 100644 index 0000000..e3c4c24 --- /dev/null +++ b/src/openultrasast/model/report.py @@ -0,0 +1,83 @@ +"""Reporting that cannot flatter itself (model-grounded-detection, Req 9.3, 11.1). + +Three disciplines survive from the deleted publish/scoring layer, and each is here because its absence caused +a real error in this project's history rather than because it is tidy: + +* **Per-slice rows, always.** A headline over mixed slices once hid that agent-written code scored 44.4% + while the hand-written development slice scored 83.3%. `render` refuses to produce anything without rows. +* **The overfitting gap beside the headline.** The distance between the development slice and the deciding + real-world one is stated, never left for the reader to compute. When a slice is missing the gap is + *absent*, not zero — reading a missing measurement as "no overfitting" is the flattering failure. +* **The rung on every finding.** A finding's strength is what established it; omitting the rung invites a + reader to treat a suspicion as a result. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from ..findings import StaticFinding + + +@dataclass(frozen=True) +class SliceRow: + """One slice's coverage. `covered` is pairs the model *split*, not pairs it flagged.""" + + slice: str + pairs: int + covered: int + + @property + def rate(self) -> float: + return self.covered / self.pairs if self.pairs else 0.0 + + +def overfitting_gap(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> float | None: + """Development-slice coverage minus the deciding slice's, or ``None`` when either is absent. + + ``None`` rather than ``0.0`` on purpose: a slice that was never measured must not read as agreement + between them. + """ + by_name = {row.slice: row for row in rows} + if development not in by_name or deciding not in by_name: + return None + return by_name[development].rate - by_name[deciding].rate + + +def render(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> str: + """The report. Raises when there are no per-slice rows, because a bare aggregate is the thing to prevent.""" + if not rows: + raise ValueError("a report needs per-slice rows: there is no headline without them") + + lines = ["## Model coverage", "", "| slice | pairs | covered | coverage |", "|---|---|---|---|"] + for row in sorted(rows, key=lambda r: r.slice): + lines.append(f"| {row.slice} | {row.pairs} | {row.covered} | {row.rate:.0%} |") + + total_pairs = sum(row.pairs for row in rows) + total_covered = sum(row.covered for row in rows) + lines += ["", f"Across every slice: {total_covered}/{total_pairs} = {total_covered / total_pairs:.0%} " + "— read the rows above it, not this line."] + + gap = overfitting_gap(rows, development=development, deciding=deciding) + if gap is not None: + lines += [ + "", + f"**Overfitting gap: {gap:+.0%}** — `{development}` (development) minus `{deciding}`, " + f"which is the deciding real-world slice.", + ] + elif development or deciding: + lines += ["", f"**Overfitting gap: not measurable** — `{development}` or `{deciding}` is absent from this run."] + return "\n".join(lines) + "\n" + + +def render_findings(findings: Sequence[StaticFinding]) -> str: + """Findings with the rung that established each one (Req 5.3).""" + lines = ["## Findings", "", "| finding | location | rung | severity |", "|---|---|---|---|"] + for finding in findings: + where = f"{finding.path}:{finding.line}" if finding.line else finding.path + lines.append(f"| {finding.title} | {where} | `{finding.rung}` | {finding.severity} |") + return "\n".join(lines) + "\n" + + +__all__ = ["SliceRow", "overfitting_gap", "render", "render_findings"] diff --git a/tests/test_model_calibrate.py b/tests/test_model_calibrate.py new file mode 100644 index 0000000..b0e75b1 --- /dev/null +++ b/tests/test_model_calibrate.py @@ -0,0 +1,93 @@ +"""model-grounded-detection task 5.1: the corpus calibrates the MODEL, not an LLM vote (Req 9). + +The inversion this feature rests on. Previously a pair the detector missed was a detector failure to be +averaged away over K runs. Here a pair the model cannot split is a *named gap in the model*, with the reason +recorded, because the model is deterministic: it will miss that pair the same way every time, and the only +thing that changes it is better modelling. + +`covered` therefore means the model reached a strictly higher rung on the vulnerable side than on its fix -- +not that it flagged something. Flagging both sides equally distinguishes nothing. +""" + +from __future__ import annotations + + +def _outcome(name, family="injection", slice_name="vibe-py", vuln="model_entailed", fixed="none", reason=""): # type: ignore[no-untyped-def] + from openultrasast.model.calibrate import PairOutcome + + return PairOutcome(pair=name, family=family, slice=slice_name, vuln_rung=vuln, fixed_rung=fixed, reason=reason) + + +def test_a_pair_the_model_splits_is_covered() -> None: + from openultrasast.model.calibrate import calibrate + + report = calibrate([_outcome("a")]) + assert report.covered == ("a",) + assert report.gaps == () + + +def test_a_pair_the_model_cannot_split_is_a_named_gap_not_a_miss() -> None: + """Req 9.2: the reason travels with the gap, because a deterministic model fails the same way every time.""" + from openultrasast.model.calibrate import calibrate + + report = calibrate([_outcome("b", vuln="none", fixed="none", reason="sink_not_in_table")]) + assert report.gaps and report.gaps[0].pair == "b" + assert report.gaps[0].reason == "sink_not_in_table" + assert report.covered == () + + +def test_flagging_both_sides_equally_is_a_gap_not_a_success() -> None: + """The model saw something on the fix too, so it distinguished nothing — that is the whole metric.""" + from openultrasast.model.calibrate import calibrate + + report = calibrate([_outcome("c", vuln="model_entailed", fixed="model_entailed")]) + assert report.covered == () + assert report.gaps[0].reason, "an unsplit pair must carry a reason even when none was supplied" + + +def test_gaps_are_grouped_by_family_with_their_reasons() -> None: + from openultrasast.model.calibrate import calibrate + + report = calibrate([ + _outcome("a", family="path", vuln="none", reason="sink_not_in_table"), + _outcome("b", family="path", vuln="none", reason="sink_not_in_table"), + _outcome("c", family="injection", vuln="none", reason="source_unmodelled"), + _outcome("d", family="injection"), + ]) + by_family = report.gaps_by_family() + assert set(by_family) == {"path", "injection"} + assert by_family["path"]["pairs"] == 2 + assert by_family["path"]["reasons"] == {"sink_not_in_table": 2} + assert by_family["injection"]["pairs"] == 1 + + +def test_the_report_is_per_slice_and_names_the_deciding_one() -> None: + """Req 9.3/11.1: the real-world slice decides; a development slice must not stand in for the corpus.""" + from openultrasast.model.calibrate import calibrate + + report = calibrate([ + _outcome("a", slice_name="vibe-py"), + _outcome("b", slice_name="vfc-js", vuln="none", reason="sink_not_in_table"), + ]) + assert set(report.per_slice) == {"vibe-py", "vfc-js"} + assert report.per_slice["vibe-py"]["covered"] == 1 + assert report.per_slice["vfc-js"]["gaps"] == 1 + + +def test_the_report_refuses_a_bare_aggregate() -> None: + from openultrasast.model.calibrate import calibrate + + text = calibrate([_outcome("a"), _outcome("b", slice_name="vfc-js", vuln="none")]).to_markdown() + assert "vibe-py" in text and "vfc-js" in text, "no headline without its per-slice rows" + assert "coverage" in text.lower() + + +def test_a_cve_recipe_yields_the_parent_and_fix_commits() -> None: + """Req 9.4: the harvest seam. The fix diff is the oracle; nothing is hand-labelled.""" + from openultrasast.model.calibrate import harvest_recipe + + recipe = harvest_recipe({"repo": "org/proj", "fix_commit": "abc123", "relpath": "app.py", "function": "run"}) + assert recipe.repo == "org/proj" + assert recipe.fix_commit == "abc123" + assert recipe.parent_ref == "abc123^", "the vulnerable side is the fix commit's parent, by construction" + assert recipe.relpath == "app.py" and recipe.function == "run" diff --git a/tests/test_model_report.py b/tests/test_model_report.py new file mode 100644 index 0000000..e308d8e --- /dev/null +++ b/tests/test_model_report.py @@ -0,0 +1,79 @@ +"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1). + +Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a +real error in this project's history: + + per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the + hand-written development slice scored 83% + the overfitting gap published beside the headline, so the distance between the development slice and the + real-world one is never something a reader has to compute + the rung on every a finding's strength is what established it; a report that omits the rung invites the + finding reader to treat a suspicion as a result +""" + +from __future__ import annotations + +import pytest + + +def _slice(name, pairs, covered): # type: ignore[no-untyped-def] + from openultrasast.model.report import SliceRow + + return SliceRow(slice=name, pairs=pairs, covered=covered) + + +def test_the_report_carries_a_row_per_slice() -> None: + from openultrasast.model.report import render + + text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)]) + assert "vibe-py" in text and "vfc-js" in text + assert "30" in text and "17" in text + + +def test_a_bare_aggregate_is_refused() -> None: + """Req 11.1: no cross-slice number without the rows it was computed from.""" + from openultrasast.model.report import render + + with pytest.raises(ValueError, match="per-slice"): + render([]) + + +def test_the_overfitting_gap_is_published_beside_the_headline() -> None: + """The development slice minus the deciding real-world slice, stated rather than left to be computed.""" + from openultrasast.model.report import overfitting_gap, render + + rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)] + gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js") + assert gap == pytest.approx(0.80 - 0.20) + assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower() + assert "+60" in render(rows, development="vibe-py", deciding="vfc-js") + + +def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None: + """A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one.""" + from openultrasast.model.report import overfitting_gap + + assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None + + +def test_every_finding_shows_its_rung() -> None: + from openultrasast.findings import StaticFinding + from openultrasast.model.ladder import Rung + from openultrasast.model.report import render_findings + + finding = StaticFinding( + finding_id="r:app.py:1", path="app.py", title="t", severity="high", confidence="medium", + evidence_level="static_corroboration", rationale="why", line=1, function_name="run", + reachability_status="unknown", reachability_evidence=[], reachability_conditions=[], tags=[], + ranking_priority=1.0, rung=Rung.ENTAILED, + ) + text = render_findings([finding]) + assert "model_entailed" in text and "app.py" in text + + +def test_the_deciding_slice_is_named_in_the_report() -> None: + """Req 9.3: the real-world slice decides, and the report says which one that is.""" + from openultrasast.model.report import render + + text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js") + assert "vfc-js" in text and "decid" in text.lower()