marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 04:29:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(plane): scoring parity with the reference -- sites, ranges, triage cost

Browse Source 
agree matched declared sites through benchmarks/independent/evaluate.py,
which the runner image does not carry, so site matching would have been
"not assessed". The generator now resolves the fix's old-side ranges
(evaluate.hunks, line for line) into case.json, with the declared sites in
the set and the reference's cost, and agree.matches_declared reproduces
matches_v2; a parity test runs both over the recorded outputs of the 15
validation cases (200 probes, identical verdicts) and agree on those outputs
returns the reference's 16 of 20 and $0.022.

agree reports cost_per_candidate (plane spend) and
cost_per_candidate_with_recorded_triage; Requirement 6.2 is judged on the
latter (design.md). --runner-image re-pins the templates from ops/ax/up.sh's
pin file; the committed manifests are regenerated at the current digest.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-29 19:49:28 +02:00
co-authored by Claude Opus 5.5
parent ac296ee3a4
commit 56b1717246
42 changed files with 591 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/cli.py
+18 -4
View File
@@ -256,6 +256,13 @@ def main(argv: list[str] | None = None) -> int:
plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
plane_ws.add_argument("--run", default="validation-46", help="with --validation-set: the Run name")
plane_ws.add_argument("--plane", type=Path, default=Path("plane"), help="with --validation-set: the manifest root")
plane_ws.add_argument(
"--repos", type=Path, default=Path.home() / ".cache" / "openultrasast" / "independent",
help="with --validation-set: one clone per case id, for the fix ranges agree scores by (evaluate.hunks)",
) # fmt: skip
plane_ws.add_argument(
"--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)"
)
args = parser.parse_args(argv)
# Diagnosis needs the stage costs, and nothing configures logging, so the default root level of
@@ -1324,11 +1331,18 @@ def _plane_increment(args: argparse.Namespace) -> int:
print("--validation-set needs --candidates and --triage", file=sys.stderr)
return 2
home = str(Path.home())
shown = [str(p).replace(home, "~", 1) for p in (args.population, args.validation_set, args.candidates, args.triage)]
command = f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]} --run {args.run}"
written = increment(
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command
paths = (args.population, args.validation_set, args.candidates, args.triage, args.repos)
shown = [str(p).replace(home, "~", 1) for p in paths]
command = (
f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]}"
f" --repos {shown[4]} --run {args.run}"
)
if args.runner_image is not None:
command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}"
written = increment(
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
repos=args.repos, runner_image=args.runner_image,
) # fmt: skip
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
return 0