mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
fix(learning-harness): round zero writes down every pair it has paid for
The 176-pair vfc baseline hit its four-hour timeout and wrote nothing. Scores were accumulated in memory and persisted once at the end, so roughly four hours of paid API calls were lost. I fixed the exception path this morning -- a provider hiccup costs one pair -- and missed that a timeout, a kill or a machine going away cost all of them. Each pair is now appended to `scores.jsonl` and flushed as it is measured, and `--resume` keeps what a previous run of that exact baseline already bought. This also gives the per-pair outcomes the round-3 review asked for, which make the noise floor auditable after the fact instead of only in aggregate. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
78b4f33a8e
commit
506bbe5f5c
@@ -223,6 +223,12 @@ def main(argv: list[str] | None = None) -> int:
|
||||
command.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
command.add_argument("--slice", choices=SLICE_NAMES, default="all")
|
||||
command.add_argument("--json", action="store_true")
|
||||
if name == "baseline":
|
||||
command.add_argument(
|
||||
"--resume",
|
||||
action="store_true",
|
||||
help="keep the pairs a previous run of this baseline already measured and paid for",
|
||||
)
|
||||
if name == "candidates":
|
||||
command.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites")
|
||||
if name in {"baseline", "round", "score"}:
|
||||
@@ -1352,6 +1358,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
|
||||
slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES,
|
||||
spent_usd=spent,
|
||||
thinking=config.learning.thinking,
|
||||
resume=args.resume,
|
||||
)
|
||||
print(f"learning baseline cost: ${report.cost_usd:.4f}")
|
||||
print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}")
|
||||
|
||||
Reference in New Issue
Block a user