mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
fix(learning-harness): the published table lost a whole slice
`publish` keyed baselines by model alone, so the vibe-py run silently replaced the agent-vfc run and the table showed hand-written teaching code as if it were the corpus. That is precisely the failure the constrained-detector requirements exist to prevent, live in a published artifact. Third time today that a key was fixed in one place and not another -- `compare_baselines` got the model+slice label and `_baselines` did not, the same way the exception path got checkpointing and the timeout path did not. The table now carries a slice column and one row per slice, and the artifact is keyed by what identifies a run. Republished: agent-vfc's six families are back, including access_control at 0.000 recall on four pairs, which is the number that was missing. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
506bbe5f5c
commit
84b1ba7a72
@@ -28,10 +28,11 @@ class Baseline:
|
||||
"""One model's round-zero artifact, read back."""
|
||||
|
||||
model: str
|
||||
metrics: dict[str, Mapping[str, object]]
|
||||
floors: dict[str, Mapping[str, object]]
|
||||
cost_usd: float
|
||||
k_runs: int
|
||||
slices: tuple[str, ...] = ()
|
||||
metrics: dict[str, Mapping[str, object]] = field(default_factory=dict)
|
||||
floors: dict[str, Mapping[str, object]] = field(default_factory=dict)
|
||||
cost_usd: float = 0.0
|
||||
k_runs: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -74,6 +75,11 @@ def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date:
|
||||
return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts))
|
||||
|
||||
|
||||
def _label(model: str, slices: tuple[str, ...]) -> str:
|
||||
"""What identifies a run in the published table: the model, and the slice when it is not the default pair."""
|
||||
return f"{model} ({', '.join(slices)})" if slices else model
|
||||
|
||||
|
||||
def _baselines(learning_dir: Path) -> dict[str, Baseline]:
|
||||
root = learning_dir / "baseline"
|
||||
found: dict[str, Baseline] = {}
|
||||
@@ -82,9 +88,13 @@ def _baselines(learning_dir: Path) -> dict[str, Baseline]:
|
||||
if not report.is_file():
|
||||
continue
|
||||
payload = json.loads(report.read_text(encoding="utf-8"))
|
||||
# Keyed by model *and* slice. Keyed by model alone, a second slice silently replaced the first and the
|
||||
# published table showed one of them as if it were the corpus (Req 3.1, 3.2).
|
||||
model = str(payload.get("model", directory.name))
|
||||
found[model] = Baseline(
|
||||
slices = tuple(str(item) for item in (payload.get("slices") or ()))
|
||||
found[_label(model, slices)] = Baseline(
|
||||
model=model,
|
||||
slices=slices,
|
||||
metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)},
|
||||
floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)},
|
||||
cost_usd=float(payload.get("cost_usd", 0.0) or 0.0),
|
||||
@@ -110,10 +120,15 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map
|
||||
lines = [
|
||||
f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.",
|
||||
"",
|
||||
"| model | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair |",
|
||||
"|---|---|---|---|---|---|---|---|---|---|",
|
||||
(
|
||||
"| model | slice | family | scorable | unscorable | recall | silence | Youden "
|
||||
"| at fixed FPR | noise floor | cost per correct pair |"
|
||||
),
|
||||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||||
]
|
||||
for model, baseline in sorted(models.items()):
|
||||
for _label_key, baseline in sorted(models.items()):
|
||||
model = baseline.model
|
||||
slice_name = ", ".join(baseline.slices) or "all"
|
||||
for family in families:
|
||||
row = baseline.metrics.get(family)
|
||||
if row is None:
|
||||
@@ -126,7 +141,7 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map
|
||||
budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0
|
||||
cost = baseline.cost_usd
|
||||
lines.append(
|
||||
f"| {model} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) "
|
||||
f"| {model} | {slice_name} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) "
|
||||
f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} "
|
||||
f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user