fix(learning-harness): say why the proposer had nothing, and stop hard-coding its vendor

The meta-agent round journalled `no_proposal`. The real cause was an
authentication failure: `_model_config` hard-coded the anthropic provider
while the detector was talking to DeepSeek and no ANTHROPIC_API_KEY exists
here. The composition itself was fine -- MetaAgent constructed, evolve
started, budget $5 -- and the trace was thrown away.

`no_proposal` covered two different worlds: a proposer that considered the
facts and declined, and one that could not start at all. The journal now
carries the proposer's own reason, and the vendor comes from
`[harnessx] provider` instead of being assumed.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 22:24:40 +02:00
co-authored by Claude Opus 5
parent 84b1ba7a72
commit cde57e4293
7 changed files with 66 additions and 12 deletions
+3 -1
View File
@@ -161,7 +161,9 @@ def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace:
assert len(rounds) == 1 and rounds[0]["family"] == "injection"
# Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of
# the enum would have hidden that the round could never propose anything at all.
assert (rounds[0]["outcome"], rounds[0]["reason"]) == ("rejected", "no_proposal")
# The reason now carries *why* the proposer had nothing: "no proposal" and "could not start" are different
# worlds, and only one of them is a fixable configuration error.
assert rounds[0]["outcome"] == "rejected" and rounds[0]["reason"].startswith("no_proposal")
assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file()
+7 -2
View File
@@ -187,8 +187,13 @@ def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_
"taxonomy_version": "1",
"metrics": {
"injection": {
"family": "injection", "scorable": 10, "unscorable": {}, "recall": recall,
"silence": 1.0, "youden": recall, "hierarchical_credit": False,
"family": "injection",
"scorable": 10,
"unscorable": {},
"recall": recall,
"silence": 1.0,
"youden": recall,
"hierarchical_credit": False,
"outcomes": {"pair_correct": 5},
}
},
+2 -2
View File
@@ -68,12 +68,12 @@ def test_the_meta_agent_is_built_with_a_model_config_not_a_model_name(tmp_path:
return workspace
monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent)
monkeypatch.setattr(proposer_module, "_model_config", lambda model: f"ModelConfig({model})")
monkeypatch.setattr(proposer_module, "_model_config", lambda model, provider="anthropic": f"ModelConfig({model}@{provider})")
(tmp_path / "injection").mkdir()
(tmp_path / "injection" / "checklist.md").write_text("- old\n")
proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts())
assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"}
assert seen["inner_model"] == "ModelConfig(m)", "the agent was handed a model name where it wanted a config"
assert seen["inner_model"] == "ModelConfig(m@anthropic)", "the agent was handed a model name where it wanted a config"
assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root"
+33
View File
@@ -373,3 +373,36 @@ def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None:
out_dir=tmp_path / "out",
k_runs=1,
)
def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: # type: ignore[no-untyped-def]
"""`no_proposal` covers two different worlds: a proposer with nothing to add, and one that could not start.
Measured: the meta-agent round journalled `no_proposal` while the real cause was an authentication failure in a
provider seam pointing at the wrong vendor. A round that failed for a fixable configuration reason must not
look identical to a model that considered the facts and declined."""
class Mute:
reason = "meta_agent_failed: RuntimeError"
def propose(self, facts): # type: ignore[no-untyped-def]
del facts
return None
from openultrasast.learning.rounds import run_learning_round
record = run_learning_round(
world["cases"],
family="injection",
taxonomy=load_families(),
configs_dir=world["configs"],
proposer=Mute(),
scan_factory=_injection_improves,
model="m",
journal=world["journal"],
archive=world["archive"],
floors=world["floors"],
out_dir=world["out"],
)
assert record.outcome == "rejected"
assert record.reason == "no_proposal: meta_agent_failed: RuntimeError"