From cde57e4293c63e9a4189ca2d77bf3dfb7a617cff Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:23:34 +0200 Subject: [PATCH] fix(learning-harness): say why the proposer had nothing, and stop hard-coding its vendor The meta-agent round journalled `no_proposal`. The real cause was an authentication failure: `_model_config` hard-coded the anthropic provider while the detector was talking to DeepSeek and no ANTHROPIC_API_KEY exists here. The composition itself was fine -- MetaAgent constructed, evolve started, budget $5 -- and the trace was thrown away. `no_proposal` covered two different worlds: a proposer that considered the facts and declined, and one that could not start at all. The journal now carries the proposer's own reason, and the vendor comes from `[harnessx] provider` instead of being assumed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- src/openultrasast/cli.py | 6 ++--- src/openultrasast/learning/proposer.py | 9 ++++--- src/openultrasast/learning/rounds.py | 13 +++++++++- tests/test_learning_cli.py | 4 +++- tests/test_learning_publish.py | 9 +++++-- tests/test_learning_round4_evidence.py | 4 ++-- tests/test_learning_round_evidence.py | 33 ++++++++++++++++++++++++++ 7 files changed, 66 insertions(+), 12 deletions(-) diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 7479bfd..6d3d65b 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -1369,7 +1369,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: family=args.family, taxonomy=taxonomy, configs_dir=configs_dir, - proposer=_proposer(args, configs_dir, model), + proposer=_proposer(args, configs_dir, model, config.harnessx.provider), scan_factory=factory, model=model, journal=journal, @@ -1402,13 +1402,13 @@ def _label(case: PairCase, taxonomy: object) -> str: return _family_of(case, taxonomy) # type: ignore[arg-type] -def _proposer(args: argparse.Namespace, configs_dir: Path, model: str) -> Proposer: +def _proposer(args: argparse.Namespace, configs_dir: Path, model: str, provider: str = "anthropic") -> Proposer: """`--proposals FILE` runs a recorded sequence offline; without it the round asks the meta-agent.""" from .learning.proposer import HarnessXProposer, Proposal, ScriptedProposer path = getattr(args, "proposals", None) if path is None: - return HarnessXProposer(configs_dir=configs_dir, model=model) + return HarnessXProposer(configs_dir=configs_dir, model=model, provider=provider) rows = json.loads(Path(path).read_text(encoding="utf-8")) return ScriptedProposer( [ diff --git a/src/openultrasast/learning/proposer.py b/src/openultrasast/learning/proposer.py index 3fce6ce..7881657 100644 --- a/src/openultrasast/learning/proposer.py +++ b/src/openultrasast/learning/proposer.py @@ -177,6 +177,9 @@ class HarnessXProposer: configs_dir: Path model: str + # The vendor the meta-agent talks to. Hard-coding anthropic here sent it to a provider with no key configured + # while the detector was talking to DeepSeek, and the round journalled that as "no proposal". + provider: str = "anthropic" reason: str = "" agent: MetaAgentCall | None = None # injected in tests; None composes MetaAgent behind harness_ext @@ -190,7 +193,7 @@ class HarnessXProposer: self.reason = "harnessx_unavailable" return None try: - call = _meta_agent_call() + call = _meta_agent_call(self.provider) except Exception: # noqa: BLE001 — a proposer that cannot start proposes nothing and says so self.reason = "harnessx_unavailable" return None @@ -263,7 +266,7 @@ def _model_config(model: str, provider: str = "anthropic") -> object: return ModelConfig(main=build_provider(model, provider)) -def _meta_agent_call() -> MetaAgentCall: +def _meta_agent_call(provider: str = "anthropic") -> MetaAgentCall: """Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy. `evolve` is a coroutine and returns a HarnessX config path; what this proposer takes from it is the state of the @@ -279,7 +282,7 @@ def _meta_agent_call() -> MetaAgentCall: trajectories = workspace.parent / "trajectories" trajectories.mkdir(parents=True, exist_ok=True) (trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8") - agent = _meta_agent_class()(inner_model=_model_config(model), allowed_write_roots=(workspace,)) + agent = _meta_agent_class()(inner_model=_model_config(model, provider), allowed_write_roots=(workspace,)) output = workspace.parent / "meta-out" output.mkdir(parents=True, exist_ok=True) asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output)) diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py index cc08b1e..b91003e 100644 --- a/src/openultrasast/learning/rounds.py +++ b/src/openultrasast/learning/rounds.py @@ -418,7 +418,18 @@ def run_learning_round( proposal = proposer.propose(facts) # type: ignore[attr-defined] record = _round_record(round_number, family, proposal, taxonomy, config) if proposal is None: - return _finish(journal, directory, record, outcome="rejected", reason="no_proposal", cost=spent, trajectories=trajectories) + # `no_proposal` covers two different worlds: a proposer that considered the facts and declined, and one + # that could not start at all. Losing that distinction hides a fixable configuration error as a quiet round. + why = str(getattr(proposer, "reason", "") or "") + return _finish( + journal, + directory, + record, + outcome="rejected", + reason=f"no_proposal: {why}" if why else "no_proposal", + cost=spent, + trajectories=trajectories, + ) refusal = refuse_proposal(proposal, config) if refusal is not None: return _finish( diff --git a/tests/test_learning_cli.py b/tests/test_learning_cli.py index ef9658b..b33e45a 100644 --- a/tests/test_learning_cli.py +++ b/tests/test_learning_cli.py @@ -161,7 +161,9 @@ def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace: assert len(rounds) == 1 and rounds[0]["family"] == "injection" # Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of # the enum would have hidden that the round could never propose anything at all. - assert (rounds[0]["outcome"], rounds[0]["reason"]) == ("rejected", "no_proposal") + # The reason now carries *why* the proposer had nothing: "no proposal" and "could not start" are different + # worlds, and only one of them is a fixable configuration error. + assert rounds[0]["outcome"] == "rejected" and rounds[0]["reason"].startswith("no_proposal") assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file() diff --git a/tests/test_learning_publish.py b/tests/test_learning_publish.py index def4b9f..07601e2 100644 --- a/tests/test_learning_publish.py +++ b/tests/test_learning_publish.py @@ -187,8 +187,13 @@ def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_ "taxonomy_version": "1", "metrics": { "injection": { - "family": "injection", "scorable": 10, "unscorable": {}, "recall": recall, - "silence": 1.0, "youden": recall, "hierarchical_credit": False, + "family": "injection", + "scorable": 10, + "unscorable": {}, + "recall": recall, + "silence": 1.0, + "youden": recall, + "hierarchical_credit": False, "outcomes": {"pair_correct": 5}, } }, diff --git a/tests/test_learning_round4_evidence.py b/tests/test_learning_round4_evidence.py index bb971b4..3d09877 100644 --- a/tests/test_learning_round4_evidence.py +++ b/tests/test_learning_round4_evidence.py @@ -68,12 +68,12 @@ def test_the_meta_agent_is_built_with_a_model_config_not_a_model_name(tmp_path: return workspace monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent) - monkeypatch.setattr(proposer_module, "_model_config", lambda model: f"ModelConfig({model})") + monkeypatch.setattr(proposer_module, "_model_config", lambda model, provider="anthropic": f"ModelConfig({model}@{provider})") (tmp_path / "injection").mkdir() (tmp_path / "injection" / "checklist.md").write_text("- old\n") proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts()) assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"} - assert seen["inner_model"] == "ModelConfig(m)", "the agent was handed a model name where it wanted a config" + assert seen["inner_model"] == "ModelConfig(m@anthropic)", "the agent was handed a model name where it wanted a config" assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root" diff --git a/tests/test_learning_round_evidence.py b/tests/test_learning_round_evidence.py index 3d4cbdc..b6a1d1c 100644 --- a/tests/test_learning_round_evidence.py +++ b/tests/test_learning_round_evidence.py @@ -373,3 +373,36 @@ def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None: out_dir=tmp_path / "out", k_runs=1, ) + + +def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: # type: ignore[no-untyped-def] + """`no_proposal` covers two different worlds: a proposer with nothing to add, and one that could not start. + + Measured: the meta-agent round journalled `no_proposal` while the real cause was an authentication failure in a + provider seam pointing at the wrong vendor. A round that failed for a fixable configuration reason must not + look identical to a model that considered the facts and declined.""" + + class Mute: + reason = "meta_agent_failed: RuntimeError" + + def propose(self, facts): # type: ignore[no-untyped-def] + del facts + return None + + from openultrasast.learning.rounds import run_learning_round + + record = run_learning_round( + world["cases"], + family="injection", + taxonomy=load_families(), + configs_dir=world["configs"], + proposer=Mute(), + scan_factory=_injection_improves, + model="m", + journal=world["journal"], + archive=world["archive"], + floors=world["floors"], + out_dir=world["out"], + ) + assert record.outcome == "rejected" + assert record.reason == "no_proposal: meta_agent_failed: RuntimeError"