mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
fix(learning-harness): a provider hiccup costs one pair, not the whole run
Round zero is the most expensive thing this harness does. An error two thirds of the way through raised straight out of `run_baseline` and took everything already measured and paid for with it, so the only honest response to a flaky endpoint was to run the whole thing again and hope. The pair is now recorded as unscorable with the reason `detector_unreachable` and the run finishes. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
fc98a48579
commit
c1ef3ca539
@@ -123,7 +123,22 @@ def run_baseline(
|
||||
config = configs.get(name)
|
||||
if config is None:
|
||||
continue # a family with no configuration is not measured rather than measured with someone else's
|
||||
scores.append(score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name))
|
||||
try:
|
||||
scores.append(score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name))
|
||||
except Exception: # noqa: BLE001 — a provider hiccup costs one pair, not the whole run
|
||||
# Round zero is the most expensive thing this harness does. Raising here threw away everything already
|
||||
# measured and paid for, so the only honest response to a flaky endpoint was to run it all again.
|
||||
scores.append(
|
||||
PairFamilyScore(
|
||||
pair=case.name,
|
||||
family=name,
|
||||
slice=case.slice,
|
||||
fix_date=case.fix_date,
|
||||
runs=(),
|
||||
outcome="unscorable",
|
||||
unscorable_reason="detector_unreachable",
|
||||
)
|
||||
)
|
||||
metrics = aggregate(scores, taxonomy=taxonomy, cutoff=cutoff)
|
||||
floors = {name: _floor(name, block, [item for item in scores if item.family == name], k_runs) for name, block in metrics.items()}
|
||||
report = BaselineReport(
|
||||
|
||||
Reference in New Issue
Block a user