mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-20 21:49:28 +00:00
fix(ci): run ruff format, and record run 5 at 90% pair-correct
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.
Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.
Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 90.0%
leak 12.3% 6.7%
detection -- 96.7%
cost ~$0.69 $0.0068
27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).
One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.
The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
14ca0a96ce
commit
fe45081b43
@@ -24,6 +24,7 @@ class FamilyGaps(TypedDict):
|
||||
pairs: int
|
||||
reasons: dict[str, int]
|
||||
|
||||
|
||||
# Ascending, matching `ladder.Rung`. A pair is split when the vulnerable side outranks its fix.
|
||||
_ORDER = {"none": 0, "": 0, "suspicion": 0, "model_corroborated": 1, "model_entailed": 2, "execution_confirmed": 3}
|
||||
|
||||
@@ -82,15 +83,18 @@ class CalibrationReport:
|
||||
|
||||
def to_markdown(self) -> str:
|
||||
"""Per-slice rows first; there is no headline here that is not backed by them (Req 9.3, 11.1)."""
|
||||
lines = ["## Model calibration", "", "Coverage is a pair the model *splits*: a strictly higher rung on",
|
||||
"the vulnerable side than on its fix. Everything else is a named gap.", "",
|
||||
"| slice | pairs | covered | gaps | coverage |", "|---|---|---|---|---|"]
|
||||
lines = [
|
||||
"## Model calibration",
|
||||
"",
|
||||
"Coverage is a pair the model *splits*: a strictly higher rung on",
|
||||
"the vulnerable side than on its fix. Everything else is a named gap.",
|
||||
"",
|
||||
"| slice | pairs | covered | gaps | coverage |",
|
||||
"|---|---|---|---|---|",
|
||||
]
|
||||
for slice_name, block in sorted(self.per_slice.items()):
|
||||
pairs = block["pairs"] or 1
|
||||
lines.append(
|
||||
f"| {slice_name} | {block['pairs']} | {block['covered']} | {block['gaps']} | "
|
||||
f"{block['covered'] / pairs:.0%} |"
|
||||
)
|
||||
lines.append(f"| {slice_name} | {block['pairs']} | {block['covered']} | {block['gaps']} | {block['covered'] / pairs:.0%} |")
|
||||
by_family = self.gaps_by_family()
|
||||
if by_family:
|
||||
lines += ["", "### Gaps by family", ""]
|
||||
@@ -116,8 +120,9 @@ def calibrate(outcomes: Iterable[PairOutcome]) -> CalibrationReport:
|
||||
if split:
|
||||
covered.append(outcome.pair)
|
||||
else:
|
||||
gaps.append(ModelGap(pair=outcome.pair, family=outcome.family, slice=outcome.slice,
|
||||
reason=outcome.reason or _default_reason(outcome)))
|
||||
gaps.append(
|
||||
ModelGap(pair=outcome.pair, family=outcome.family, slice=outcome.slice, reason=outcome.reason or _default_reason(outcome))
|
||||
)
|
||||
return CalibrationReport(
|
||||
covered=tuple(sorted(covered)),
|
||||
gaps=tuple(sorted(gaps, key=lambda g: (g.family, g.pair))),
|
||||
|
||||
Reference in New Issue
Block a user