From 84b1ba7a722f5af0d531710e314ceeff44f71952 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:44:56 +0200 Subject: [PATCH] fix(learning-harness): the published table lost a whole slice `publish` keyed baselines by model alone, so the vibe-py run silently replaced the agent-vfc run and the table showed hand-written teaching code as if it were the corpus. That is precisely the failure the constrained-detector requirements exist to prevent, live in a published artifact. Third time today that a key was fixed in one place and not another -- `compare_baselines` got the model+slice label and `_baselines` did not, the same way the exception path got checkpointing and the timeout path did not. The table now carries a slice column and one row per slice, and the artifact is keyed by what identifies a run. Republished: agent-vfc's six families are back, including access_control at 0.000 recall on four pairs, which is the number that was missing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/steering/roadmap.md | 29 +- .../2026-09-06-learning-families.json | 626 ++++++++++++++++++ src/openultrasast/learning/publish.py | 33 +- tests/test_learning_publish.py | 39 +- 4 files changed, 715 insertions(+), 12 deletions(-) create mode 100644 benchmarks/measurements/2026-09-06-learning-families.json diff --git a/.kiro/steering/roadmap.md b/.kiro/steering/roadmap.md index a2b342a..e56510c 100644 --- a/.kiro/steering/roadmap.md +++ b/.kiro/steering/roadmap.md @@ -94,8 +94,33 @@ sast: java-sqli moved MISS→LEAK (coverage counted on both sides; the fixed twi ## Classified detector numbers -Generated by `uv run ousast learning publish`. +Generated by `uv run ousast learning publish` from the artifacts committed beside these numbers. -No baseline has been recorded yet, so there is nothing to publish. Run round zero first; a number without the run that produced it is a claim, not a measurement. +| model | slice | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair | +|---|---|---|---|---|---|---|---|---|---|---| +| deepseek-v4-flash | agent-vfc | access_control | 4 (taxonomy 1) | known_limit:client_side_caller: 1, unresolved_label: 1 | 0.000 | 1.000 | 0.000 | 0.000 | budget 4 | - | +| deepseek-v4-flash | agent-vfc | config_secrets | 6 (taxonomy 1) | none | 0.333 | 0.833 | 0.167 | 0.000 | budget 2 | 0.95 | +| deepseek-v4-flash | agent-vfc | injection | 2 (taxonomy 1) | none | 0.500 | 0.500 | 0.000 | 0.000 | budget 1 | - | +| deepseek-v4-flash | agent-vfc | path | 3 (taxonomy 1) | none | 1.000 | 0.667 | 0.667 | 0.000 | budget 3 | 0.48 | +| deepseek-v4-flash | agent-vfc | prototype | 7 (taxonomy 1) | none | 0.429 | 1.000 | 0.429 | 0.429 | budget 5 | 0.32 | +| deepseek-v4-flash | agent-vfc | untrusted_destination | 5 (taxonomy 1) | none | 0.600 | 0.600 | 0.200 | 0.000 | budget 3 | 0.95 | +| deepseek-v4-flash | vibe-py | access_control | 3 (taxonomy 1) | identical_twin: 1 | 1.000 | 1.000 | 1.000 | 1.000 | budget 3 | 0.23 | +| deepseek-v4-flash | vibe-py | config_secrets | 3 (taxonomy 1) | none | 1.000 | 1.000 | 1.000 | 1.000 | budget 1 | 0.23 | +| deepseek-v4-flash | vibe-py | deserialization | 2 (taxonomy 1) | identical_twin: 1 | 1.000 | 1.000 | 1.000 | 1.000 | budget 0 | 0.34 | +| deepseek-v4-flash | vibe-py | injection | 20 (taxonomy 1) | identical_twin: 2 | 0.750 | 0.900 | 0.650 | 0.750 | budget 11 | 0.05 | +| deepseek-v4-flash | vibe-py | output_encoding | 1 (taxonomy 1) | identical_twin: 1 | 1.000 | 1.000 | 1.000 | 1.000 | budget 1 | 0.69 | +| deepseek-v4-flash | vibe-py | path | 1 (taxonomy 1) | none | 1.000 | 1.000 | 1.000 | 1.000 | budget 1 | 0.69 | +| deepseek-v4-pro | vibe-py | access_control | 3 (taxonomy 1) | identical_twin: 1 | 0.000 | 1.000 | 0.000 | 0.000 | budget 2 | - | +| deepseek-v4-pro | vibe-py | config_secrets | 3 (taxonomy 1) | none | 1.000 | 1.000 | 1.000 | 1.000 | budget 2 | 0.16 | +| deepseek-v4-pro | vibe-py | deserialization | 2 (taxonomy 1) | identical_twin: 1 | 1.000 | 1.000 | 1.000 | 1.000 | budget 1 | 0.24 | +| deepseek-v4-pro | vibe-py | injection | 20 (taxonomy 1) | identical_twin: 2 | 0.350 | 0.900 | 0.250 | 0.350 | budget 14 | 0.09 | +| deepseek-v4-pro | vibe-py | output_encoding | 1 (taxonomy 1) | identical_twin: 1 | 1.000 | 1.000 | 1.000 | 1.000 | budget 0 | 0.47 | +| deepseek-v4-pro | vibe-py | path | 1 (taxonomy 1) | none | 1.000 | 1.000 | 1.000 | 1.000 | budget 0 | 0.47 | + +No family in this taxonomy declares a parent, so hierarchical partial credit could not apply to any figure above. + +No detector cutoff is configured, so no post-cutoff slice is reported; every figure above is over the whole corpus. + +Mechanism lever on the vibe-py holdout: detected 5 of 17 at Youden -0.0588 once only train-split pairs teach, against detected 7 at Youden 0.0588 when every pair taught. The difference is what holdout pairs stopped teaching. diff --git a/benchmarks/measurements/2026-09-06-learning-families.json b/benchmarks/measurements/2026-09-06-learning-families.json new file mode 100644 index 0000000..adb2757 --- /dev/null +++ b/benchmarks/measurements/2026-09-06-learning-families.json @@ -0,0 +1,626 @@ +{ + "command": "uv run ousast learning publish", + "floors": { + "deepseek-v4-flash (agent-vfc)": { + "access_control": { + "budget_flips": 4, + "family": "access_control", + "flaky_pairs": 4, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 1.0, + "pairs": 4 + }, + "config_secrets": { + "budget_flips": 2, + "family": "config_secrets", + "flaky_pairs": 2, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.3333333333333333, + "pairs": 6 + }, + "injection": { + "budget_flips": 1, + "family": "injection", + "flaky_pairs": 1, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.5, + "pairs": 2 + }, + "path": { + "budget_flips": 3, + "family": "path", + "flaky_pairs": 3, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 1.0, + "pairs": 3 + }, + "prototype": { + "budget_flips": 5, + "family": "prototype", + "flaky_pairs": 5, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.7142857142857143, + "pairs": 7 + }, + "untrusted_destination": { + "budget_flips": 3, + "family": "untrusted_destination", + "flaky_pairs": 3, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.6, + "pairs": 5 + } + }, + "deepseek-v4-flash (vibe-py)": { + "access_control": { + "budget_flips": 3, + "family": "access_control", + "flaky_pairs": 3, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 1.0, + "pairs": 3 + }, + "config_secrets": { + "budget_flips": 1, + "family": "config_secrets", + "flaky_pairs": 1, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.3333333333333333, + "pairs": 3 + }, + "deserialization": { + "budget_flips": 0, + "family": "deserialization", + "flaky_pairs": 0, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.0, + "pairs": 2 + }, + "injection": { + "budget_flips": 11, + "family": "injection", + "flaky_pairs": 11, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.55, + "pairs": 20 + }, + "output_encoding": { + "budget_flips": 1, + "family": "output_encoding", + "flaky_pairs": 1, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 1.0, + "pairs": 1 + }, + "path": { + "budget_flips": 1, + "family": "path", + "flaky_pairs": 1, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 1.0, + "pairs": 1 + } + }, + "deepseek-v4-pro (vibe-py)": { + "access_control": { + "budget_flips": 2, + "family": "access_control", + "flaky_pairs": 2, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.6666666666666666, + "pairs": 3 + }, + "config_secrets": { + "budget_flips": 2, + "family": "config_secrets", + "flaky_pairs": 2, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.6666666666666666, + "pairs": 3 + }, + "deserialization": { + "budget_flips": 1, + "family": "deserialization", + "flaky_pairs": 1, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.5, + "pairs": 2 + }, + "injection": { + "budget_flips": 14, + "family": "injection", + "flaky_pairs": 14, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.7, + "pairs": 20 + }, + "output_encoding": { + "budget_flips": 0, + "family": "output_encoding", + "flaky_pairs": 0, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.0, + "pairs": 1 + }, + "path": { + "budget_flips": 0, + "family": "path", + "flaky_pairs": 0, + "gates": true, + "k_runs": 5, + "negative_flip_rate": 0.0, + "pairs": 1 + } + } + }, + "generated": "2026-09-06", + "models": { + "deepseek-v4-flash (agent-vfc)": { + "access_control": { + "cutoff": "", + "directional_bias": -1.0, + "family": "access_control", + "fixed_fpr_recall": 0.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "both_silent": 4 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.0, + "reliable_change": false, + "scorable": 4, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "known_limit:client_side_caller": 1, + "unresolved_label": 1 + }, + "youden": 0.0 + }, + "config_secrets": { + "cutoff": "", + "directional_bias": -0.5, + "family": "config_secrets", + "fixed_fpr_recall": 0.0, + "hierarchical_credit": false, + "leak_rate": 0.16666666666666666, + "negative_flips": 0, + "outcomes": { + "both_flagged": 1, + "both_silent": 4, + "pair_correct": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.3333333333333333, + "reliable_change": false, + "scorable": 6, + "silence": 0.8333333333333334, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 0.16666666666666674 + }, + "injection": { + "cutoff": "", + "directional_bias": 0.0, + "family": "injection", + "fixed_fpr_recall": 0.0, + "hierarchical_credit": false, + "leak_rate": 0.5, + "negative_flips": 0, + "outcomes": { + "both_flagged": 1, + "both_silent": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.5, + "reliable_change": false, + "scorable": 2, + "silence": 0.5, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 0.0 + }, + "path": { + "cutoff": "", + "directional_bias": 0.3333333333333333, + "family": "path", + "fixed_fpr_recall": 0.0, + "hierarchical_credit": false, + "leak_rate": 0.3333333333333333, + "negative_flips": 0, + "outcomes": { + "both_flagged": 1, + "pair_correct": 2 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 3, + "silence": 0.6666666666666666, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 0.6666666666666665 + }, + "prototype": { + "cutoff": "", + "directional_bias": -0.5714285714285714, + "family": "prototype", + "fixed_fpr_recall": 0.42857142857142855, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "both_silent": 4, + "pair_correct": 3 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.42857142857142855, + "reliable_change": false, + "scorable": 7, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 0.4285714285714286 + }, + "untrusted_destination": { + "cutoff": "", + "directional_bias": 0.0, + "family": "untrusted_destination", + "fixed_fpr_recall": 0.0, + "hierarchical_credit": false, + "leak_rate": 0.4, + "negative_flips": 0, + "outcomes": { + "both_flagged": 2, + "both_silent": 2, + "pair_correct": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.6, + "reliable_change": false, + "scorable": 5, + "silence": 0.6, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 0.19999999999999996 + } + }, + "deepseek-v4-flash (vibe-py)": { + "access_control": { + "cutoff": "", + "directional_bias": 0.0, + "family": "access_control", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 3 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 3, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 1 + }, + "youden": 1.0 + }, + "config_secrets": { + "cutoff": "", + "directional_bias": 0.0, + "family": "config_secrets", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 3 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 3, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 1.0 + }, + "deserialization": { + "cutoff": "", + "directional_bias": 0.0, + "family": "deserialization", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 2 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 2, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 1 + }, + "youden": 1.0 + }, + "injection": { + "cutoff": "", + "directional_bias": -0.15, + "family": "injection", + "fixed_fpr_recall": 0.75, + "hierarchical_credit": false, + "leak_rate": 0.1, + "negative_flips": 0, + "outcomes": { + "both_flagged": 2, + "both_silent": 5, + "pair_correct": 13 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.75, + "reliable_change": false, + "scorable": 20, + "silence": 0.9, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 2 + }, + "youden": 0.6499999999999999 + }, + "output_encoding": { + "cutoff": "", + "directional_bias": 0.0, + "family": "output_encoding", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 1, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 1 + }, + "youden": 1.0 + }, + "path": { + "cutoff": "", + "directional_bias": 0.0, + "family": "path", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 1, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 1.0 + } + }, + "deepseek-v4-pro (vibe-py)": { + "access_control": { + "cutoff": "", + "directional_bias": -1.0, + "family": "access_control", + "fixed_fpr_recall": 0.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "both_silent": 3 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.0, + "reliable_change": false, + "scorable": 3, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 1 + }, + "youden": 0.0 + }, + "config_secrets": { + "cutoff": "", + "directional_bias": 0.0, + "family": "config_secrets", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 3 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 3, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 1.0 + }, + "deserialization": { + "cutoff": "", + "directional_bias": 0.0, + "family": "deserialization", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 2 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 2, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 1 + }, + "youden": 1.0 + }, + "injection": { + "cutoff": "", + "directional_bias": -0.55, + "family": "injection", + "fixed_fpr_recall": 0.35, + "hierarchical_credit": false, + "leak_rate": 0.1, + "negative_flips": 0, + "outcomes": { + "both_flagged": 2, + "both_silent": 13, + "pair_correct": 5 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 0.35, + "reliable_change": false, + "scorable": 20, + "silence": 0.9, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 2 + }, + "youden": 0.25 + }, + "output_encoding": { + "cutoff": "", + "directional_bias": 0.0, + "family": "output_encoding", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 1, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": { + "identical_twin": 1 + }, + "youden": 1.0 + }, + "path": { + "cutoff": "", + "directional_bias": 0.0, + "family": "path", + "fixed_fpr_recall": 1.0, + "hierarchical_credit": false, + "leak_rate": 0.0, + "negative_flips": 0, + "outcomes": { + "pair_correct": 1 + }, + "p_value": null, + "positive_flips": 0, + "post_cutoff": {}, + "recall": 1.0, + "reliable_change": false, + "scorable": 1, + "silence": 1.0, + "taxonomy_version": "1", + "undated": 0, + "unscorable": {}, + "youden": 1.0 + } + } + } +} diff --git a/src/openultrasast/learning/publish.py b/src/openultrasast/learning/publish.py index c31b725..afa9550 100644 --- a/src/openultrasast/learning/publish.py +++ b/src/openultrasast/learning/publish.py @@ -28,10 +28,11 @@ class Baseline: """One model's round-zero artifact, read back.""" model: str - metrics: dict[str, Mapping[str, object]] - floors: dict[str, Mapping[str, object]] - cost_usd: float - k_runs: int + slices: tuple[str, ...] = () + metrics: dict[str, Mapping[str, object]] = field(default_factory=dict) + floors: dict[str, Mapping[str, object]] = field(default_factory=dict) + cost_usd: float = 0.0 + k_runs: int = 0 @dataclass(frozen=True) @@ -74,6 +75,11 @@ def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date: return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts)) +def _label(model: str, slices: tuple[str, ...]) -> str: + """What identifies a run in the published table: the model, and the slice when it is not the default pair.""" + return f"{model} ({', '.join(slices)})" if slices else model + + def _baselines(learning_dir: Path) -> dict[str, Baseline]: root = learning_dir / "baseline" found: dict[str, Baseline] = {} @@ -82,9 +88,13 @@ def _baselines(learning_dir: Path) -> dict[str, Baseline]: if not report.is_file(): continue payload = json.loads(report.read_text(encoding="utf-8")) + # Keyed by model *and* slice. Keyed by model alone, a second slice silently replaced the first and the + # published table showed one of them as if it were the corpus (Req 3.1, 3.2). model = str(payload.get("model", directory.name)) - found[model] = Baseline( + slices = tuple(str(item) for item in (payload.get("slices") or ())) + found[_label(model, slices)] = Baseline( model=model, + slices=slices, metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)}, floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)}, cost_usd=float(payload.get("cost_usd", 0.0) or 0.0), @@ -110,10 +120,15 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map lines = [ f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.", "", - "| model | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair |", - "|---|---|---|---|---|---|---|---|---|---|", + ( + "| model | slice | family | scorable | unscorable | recall | silence | Youden " + "| at fixed FPR | noise floor | cost per correct pair |" + ), + "|---|---|---|---|---|---|---|---|---|---|---|", ] - for model, baseline in sorted(models.items()): + for _label_key, baseline in sorted(models.items()): + model = baseline.model + slice_name = ", ".join(baseline.slices) or "all" for family in families: row = baseline.metrics.get(family) if row is None: @@ -126,7 +141,7 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0 cost = baseline.cost_usd lines.append( - f"| {model} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) " + f"| {model} | {slice_name} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) " f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} " f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |" ) diff --git a/tests/test_learning_publish.py b/tests/test_learning_publish.py index e783111..def4b9f 100644 --- a/tests/test_learning_publish.py +++ b/tests/test_learning_publish.py @@ -153,7 +153,8 @@ def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path: written = sorted(path.name for path in measurements.iterdir()) assert any(name.endswith("-learning-families.json") for name in written) payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text()) - assert payload["models"]["deepseek-v4-flash"]["injection"]["scorable"] == 10 + # keyed by what identifies the run, so two slices of one model cannot collapse into one entry + assert payload["models"]["deepseek-v4-flash (vibe-py)"]["injection"]["scorable"] == 10 assert payload["command"] @@ -165,3 +166,39 @@ def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_p report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap) assert report.families == () and "no baseline" in report.table.lower() assert "learning-harness:begin" in roadmap.read_text() + + +def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_path: Path) -> None: + """Keyed by model alone, the vibe-py run silently replaced the agent-vfc run and the published table showed + hand-written teaching code as if it were the corpus — the exact failure Req 3 exists to prevent.""" + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + for key, slice_name, recall in (("deepseek-v4-flash-vibe-py", "vibe-py", 0.833), ("deepseek-v4-flash-agent-vfc", "agent-vfc", 0.444)): + directory = out / "baseline" / key + directory.mkdir(parents=True) + (directory / "report.json").write_text( + json.dumps( + { + "model": "deepseek-v4-flash", + "slices": [slice_name], + "k_runs": 5, + "cost_usd": 1.0, + "taxonomy_version": "1", + "metrics": { + "injection": { + "family": "injection", "scorable": 10, "unscorable": {}, "recall": recall, + "silence": 1.0, "youden": recall, "hierarchical_credit": False, + "outcomes": {"pair_correct": 5}, + } + }, + "floors": {}, + } + ) + ) + roadmap = tmp_path / "roadmap.md" + roadmap.write_text("# Roadmap\n") + report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) + assert "vibe-py" in report.table and "agent-vfc" in report.table + assert report.table.count("| injection |") == 2, "one row per slice, not one row that survived" + assert "0.833" in report.table and "0.444" in report.table