mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
feat(constrained-detector): the ceiling, measured offline across every slice
Group 1 complete. Tasks 1.2 and 1.3: `candidates.ceiling`, `ousast learning
candidates`, and the stop-or-go reading. No model is called anywhere in this
group, so the condition that could have ended the feature was reachable for
free.
Verdict: go.
agent-vfc (deciding) 89.7% ceiling against 44.4% round-zero detection
vfc (deciding) 84.7%, baseline still running, non-gating
vfc-js (deciding) 52.9% -- see below
github (deciding) 100.0%
vibe-py (development)100.0% against 83.3%
The margin is largest on agent-vfc, which is the slice the tool is actually
for.
The measurement forced a shape correction before the detector existed.
config_secrets first read 63.6% with a bind-and-literal shape, because a
permissive default is as often an argument to a call -- cors({origin: '*'}),
app.use(session({secure: false})) -- as it is a binding. Adding call sites to
that family's shape took it to 100% and recovered four agent-vfc rows.
Carry-forward, and not this feature's to fix: vfc-js sits at 52.9% because
eight of seventeen rows label a function the JavaScript CST names `<anon>` --
`module.exports.publish = function () {}` and its relatives. That is a
function-naming gap, it caps the existing hunter on that slice exactly as
much, and it is the same class as the `_declarator_start` defects found in
learning-harness task 4.1.
The ruleset arm of `ceiling` is kept so the 12.1% that rejected it stays a
measurement rather than a remembered number.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
8d771901bf
commit
78b4f33a8e
@@ -213,6 +213,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
("classify", "classify every labeled pair and report the classifier against itself"),
|
||||
("score", "score a catalog by family, with the denominator every number was computed over"),
|
||||
("baseline", "round zero: clone one detector into every family and measure each against itself"),
|
||||
("candidates", "offline: what a candidate-driven detector could find, per family and per slice"),
|
||||
("round", "one evolve round: propose one change for one family and let the evidence decide"),
|
||||
("publish", "regenerate every published number from the artifacts that produced it"),
|
||||
):
|
||||
@@ -222,6 +223,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
command.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
command.add_argument("--slice", choices=SLICE_NAMES, default="all")
|
||||
command.add_argument("--json", action="store_true")
|
||||
if name == "candidates":
|
||||
command.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites")
|
||||
if name in {"baseline", "round", "score"}:
|
||||
command.add_argument("--k-runs", type=int, default=None, help="runs per pair; never fewer than three")
|
||||
command.add_argument("--model", default=None, help="detector model; artifacts are keyed by it")
|
||||
@@ -1238,6 +1241,26 @@ def _learning(args: argparse.Namespace) -> int:
|
||||
print(f" artifact {artifact}")
|
||||
return 0
|
||||
cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice))
|
||||
if args.learning_command == "candidates":
|
||||
# No model is called anywhere in this command. The ceiling is a property of the corpus and the enumerator,
|
||||
# so it can end this feature before a cent is spent on it (constrained-detector task 1.3).
|
||||
from .learning.candidates import ceiling
|
||||
|
||||
coverage = ceiling(cases, taxonomy=taxonomy, generator=args.generator)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
payload = coverage.to_dict()
|
||||
(out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
if args.json:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
for family, block in sorted(coverage.per_family.items()):
|
||||
print(
|
||||
f"learning candidates {family}: {block['with_candidate']}/{block['pairs']} "
|
||||
f"= {_as_float(block['ceiling']):.1%} ceiling, sites {block['sites']}"
|
||||
)
|
||||
for slice_name, block in sorted(coverage.per_slice.items()):
|
||||
print(f" slice {slice_name}: {_as_float(block['ceiling']):.1%} over {block['pairs']} pairs")
|
||||
return 0
|
||||
if args.learning_command == "classify":
|
||||
measured = measure_classifier(cases, taxonomy, client=None)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
@@ -1358,6 +1381,10 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
|
||||
return 0
|
||||
|
||||
|
||||
def _as_float(value: object) -> float:
|
||||
return float(value) if isinstance(value, int | float) else 0.0
|
||||
|
||||
|
||||
def _label(case: PairCase, taxonomy: object) -> str:
|
||||
"""The family that routes this pair: the maintainer's label when there is one, else the classifier's answer.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user