marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 19:59:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(constrained-detector): the ceiling, measured offline across every slice

Browse Source 
Group 1 complete. Tasks 1.2 and 1.3: `candidates.ceiling`, `ousast learning
candidates`, and the stop-or-go reading. No model is called anywhere in this
group, so the condition that could have ended the feature was reachable for
free.

Verdict: go.

  agent-vfc (deciding)  89.7% ceiling against 44.4% round-zero detection
  vfc (deciding)        84.7%, baseline still running, non-gating
  vfc-js (deciding)     52.9% -- see below
  github (deciding)    100.0%
  vibe-py (development)100.0% against 83.3%

The margin is largest on agent-vfc, which is the slice the tool is actually
for.

The measurement forced a shape correction before the detector existed.
config_secrets first read 63.6% with a bind-and-literal shape, because a
permissive default is as often an argument to a call -- cors({origin: '*'}),
app.use(session({secure: false})) -- as it is a binding. Adding call sites to
that family's shape took it to 100% and recovered four agent-vfc rows.

Carry-forward, and not this feature's to fix: vfc-js sits at 52.9% because
eight of seventeen rows label a function the JavaScript CST names `<anon>` --
`module.exports.publish = function () {}` and its relatives. That is a
function-naming gap, it caps the existing hunter on that slice exactly as
much, and it is the same class as the `_declarator_start` defects found in
learning-harness task 4.1.

The ruleset arm of `ceiling` is kept so the 12.1% that rejected it stays a
measurement rather than a remembered number.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 18:44:19 +02:00
co-authored by Claude Opus 5
parent 8d771901bf
commit 78b4f33a8e
6 changed files with 613 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/cli.py
+27
View File
@@ -213,6 +213,7 @@ def main(argv: list[str] | None = None) -> int:
("classify", "classify every labeled pair and report the classifier against itself"),
("score", "score a catalog by family, with the denominator every number was computed over"),
("baseline", "round zero: clone one detector into every family and measure each against itself"),
("candidates", "offline: what a candidate-driven detector could find, per family and per slice"),
("round", "one evolve round: propose one change for one family and let the evidence decide"),
("publish", "regenerate every published number from the artifacts that produced it"),
):
@@ -222,6 +223,8 @@ def main(argv: list[str] | None = None) -> int:
command.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
command.add_argument("--slice", choices=SLICE_NAMES, default="all")
command.add_argument("--json", action="store_true")
if name == "candidates":
command.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites")
if name in {"baseline", "round", "score"}:
command.add_argument("--k-runs", type=int, default=None, help="runs per pair; never fewer than three")
command.add_argument("--model", default=None, help="detector model; artifacts are keyed by it")
@@ -1238,6 +1241,26 @@ def _learning(args: argparse.Namespace) -> int:
print(f" artifact {artifact}")
return 0
cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice))
if args.learning_command == "candidates":
# No model is called anywhere in this command. The ceiling is a property of the corpus and the enumerator,
# so it can end this feature before a cent is spent on it (constrained-detector task 1.3).
from .learning.candidates import ceiling
coverage = ceiling(cases, taxonomy=taxonomy, generator=args.generator)
out.mkdir(parents=True, exist_ok=True)
payload = coverage.to_dict()
(out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
if args.json:
print(json.dumps(payload, indent=2, sort_keys=True))
else:
for family, block in sorted(coverage.per_family.items()):
print(
f"learning candidates {family}: {block['with_candidate']}/{block['pairs']} "
f"= {_as_float(block['ceiling']):.1%} ceiling, sites {block['sites']}"
)
for slice_name, block in sorted(coverage.per_slice.items()):
print(f" slice {slice_name}: {_as_float(block['ceiling']):.1%} over {block['pairs']} pairs")
return 0
if args.learning_command == "classify":
measured = measure_classifier(cases, taxonomy, client=None)
out.mkdir(parents=True, exist_ok=True)
@@ -1358,6 +1381,10 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
return 0
def _as_float(value: object) -> float:
return float(value) if isinstance(value, int | float) else 0.0
def _label(case: PairCase, taxonomy: object) -> str:
"""The family that routes this pair: the maintainer's label when there is one, else the classifier's answer.
src/openultrasast/learning/candidates.py
+96 -2
View File
@@ -33,7 +33,10 @@ FAMILY_SHAPE: dict[str, tuple[CandidateKind, ...]] = {
"memory": ("call",),
"prototype": ("call", "bind"),
"access_control": ("operation",),
"config_secrets": ("bind", "config"),
# A permissive default is as often an argument to a call — `cors({origin: "*"})`, `app.use(session({secure:
# false}))` — as it is a binding. Measured: with bindings alone the shape missed `createApplication` and
# `getAllowedHosts`, whose whole bug is inside a call argument.
"config_secrets": ("call", "bind", "config"),
"unknown": ("call", "bind"),
}
@@ -232,22 +235,113 @@ def _language_of(path: str) -> str:
@dataclass(frozen=True)
class CandidateReport:
"""The ceiling: what a candidate-driven detector could find before any model is asked anything."""
"""The ceiling: what a candidate-driven detector could find before any model is asked anything.
This is the number that decides whether the design continues. A generator that cannot put a candidate inside
the labeled function can never find that pair's bug however good the judgement is, so the ceiling is published
as the limit on recall rather than discovered later as a disappointment (Req 1.3)."""
generator: str = "ir"
per_family: dict[str, dict[str, object]] = field(default_factory=dict)
per_slice: dict[str, dict[str, object]] = field(default_factory=dict)
gaps: tuple[tuple[str, str, str], ...] = () # (slice, pair, reason)
def to_dict(self) -> dict[str, object]:
return {
"generator": self.generator,
"per_family": {name: dict(block) for name, block in sorted(self.per_family.items())},
"per_slice": {name: dict(block) for name, block in sorted(self.per_slice.items())},
"gaps": [list(row) for row in self.gaps],
}
def ceiling(cases: Sequence[object], *, taxonomy: object, generator: str = "ir") -> CandidateReport:
"""Per family and per slice, the fraction of labeled functions a generator can put a candidate inside.
``generator`` is ``ir`` or ``ruleset``. The ruleset arm exists so the comparison that rejected it — 12.1%
against 96.6% on the web slices — stays a measurement rather than a remembered number."""
import tempfile
from ..pairs import _materialize_side, _overlay_scan, _quick_scan
from .rounds import _family_of
from .scoring import _spans
families: dict[str, dict[str, int]] = {}
slices: dict[str, dict[str, int]] = {}
gaps: list[tuple[str, str, str]] = []
sites: dict[str, list[int]] = {}
for case in cases:
family = _family_of(case, taxonomy) # type: ignore[arg-type]
slice_name = str(getattr(case, "slice", "") or "")
region_path = str(getattr(case, "relpath", "") or "")
with tempfile.TemporaryDirectory(prefix="ousast-ceiling-") as scratch:
root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") # type: ignore[arg-type]
if generator == "ruleset":
ranges = _overlay_scan(root).ranges
spans = _spans(case, ranges)
found = [
finding
for finding in _quick_scan(root)
if finding.line is not None and any(path == finding.path and start <= finding.line <= end for path, start, end in spans)
]
count, reason = len(found), ("" if found else "no_candidate")
else:
labeled = [str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()]
result = enumerate_candidates(
root,
_Region(region_path, next((name for name in labeled if name), None)),
family,
taxonomy=taxonomy,
limit=10**6, # the ceiling is about coverage, never about the batch bound
)
count, reason = len(result.candidates), result.reason
for bucket, key in ((families, family), (slices, slice_name)):
block = bucket.setdefault(key, {"pairs": 0, "with_candidate": 0})
block["pairs"] += 1
block["with_candidate"] += 1 if count else 0
if count:
sites.setdefault(family, []).append(count)
else:
gaps.append((slice_name, str(getattr(case, "name", "")), reason or "no_candidate"))
per_family: dict[str, dict[str, object]] = {}
per_slice: dict[str, dict[str, object]] = {}
for bucket, out in ((families, per_family), (slices, per_slice)):
for key, block in bucket.items():
pairs = block["pairs"]
row: dict[str, object] = {
"pairs": pairs,
"with_candidate": block["with_candidate"],
"ceiling": (block["with_candidate"] / pairs) if pairs else 0.0,
}
if bucket is families:
row["sites"] = _distribution(sites.get(key, []))
out[key] = row
return CandidateReport(generator=generator, per_family=per_family, per_slice=per_slice, gaps=tuple(sorted(gaps)))
def _distribution(counts: list[int]) -> dict[str, int]:
if not counts:
return {"min": 0, "median": 0, "p90": 0, "max": 0}
counts = sorted(counts)
return {
"min": counts[0],
"median": counts[len(counts) // 2],
"p90": counts[min(int(len(counts) * 0.9), len(counts) - 1)],
"max": counts[-1],
}
@dataclass(frozen=True)
class _Region:
"""The two fields `enumerate_candidates` reads, so the ceiling does not import the detector module."""
path: str
function: str | None
__all__ = [
"FAMILY_SHAPE",
"ceiling",
"MAX_CANDIDATES_PER_CALL",
"Candidate",
"CandidateKind",