mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 12:09:27 +00:00
fix(learning-harness): say why the proposer had nothing, and stop hard-coding its vendor
The meta-agent round journalled `no_proposal`. The real cause was an authentication failure: `_model_config` hard-coded the anthropic provider while the detector was talking to DeepSeek and no ANTHROPIC_API_KEY exists here. The composition itself was fine -- MetaAgent constructed, evolve started, budget $5 -- and the trace was thrown away. `no_proposal` covered two different worlds: a proposer that considered the facts and declined, and one that could not start at all. The journal now carries the proposer's own reason, and the vendor comes from `[harnessx] provider` instead of being assumed. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
84b1ba7a72
commit
cde57e4293
@@ -1369,7 +1369,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
|
|||||||
family=args.family,
|
family=args.family,
|
||||||
taxonomy=taxonomy,
|
taxonomy=taxonomy,
|
||||||
configs_dir=configs_dir,
|
configs_dir=configs_dir,
|
||||||
proposer=_proposer(args, configs_dir, model),
|
proposer=_proposer(args, configs_dir, model, config.harnessx.provider),
|
||||||
scan_factory=factory,
|
scan_factory=factory,
|
||||||
model=model,
|
model=model,
|
||||||
journal=journal,
|
journal=journal,
|
||||||
@@ -1402,13 +1402,13 @@ def _label(case: PairCase, taxonomy: object) -> str:
|
|||||||
return _family_of(case, taxonomy) # type: ignore[arg-type]
|
return _family_of(case, taxonomy) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
def _proposer(args: argparse.Namespace, configs_dir: Path, model: str) -> Proposer:
|
def _proposer(args: argparse.Namespace, configs_dir: Path, model: str, provider: str = "anthropic") -> Proposer:
|
||||||
"""`--proposals FILE` runs a recorded sequence offline; without it the round asks the meta-agent."""
|
"""`--proposals FILE` runs a recorded sequence offline; without it the round asks the meta-agent."""
|
||||||
from .learning.proposer import HarnessXProposer, Proposal, ScriptedProposer
|
from .learning.proposer import HarnessXProposer, Proposal, ScriptedProposer
|
||||||
|
|
||||||
path = getattr(args, "proposals", None)
|
path = getattr(args, "proposals", None)
|
||||||
if path is None:
|
if path is None:
|
||||||
return HarnessXProposer(configs_dir=configs_dir, model=model)
|
return HarnessXProposer(configs_dir=configs_dir, model=model, provider=provider)
|
||||||
rows = json.loads(Path(path).read_text(encoding="utf-8"))
|
rows = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||||
return ScriptedProposer(
|
return ScriptedProposer(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -177,6 +177,9 @@ class HarnessXProposer:
|
|||||||
|
|
||||||
configs_dir: Path
|
configs_dir: Path
|
||||||
model: str
|
model: str
|
||||||
|
# The vendor the meta-agent talks to. Hard-coding anthropic here sent it to a provider with no key configured
|
||||||
|
# while the detector was talking to DeepSeek, and the round journalled that as "no proposal".
|
||||||
|
provider: str = "anthropic"
|
||||||
reason: str = ""
|
reason: str = ""
|
||||||
agent: MetaAgentCall | None = None # injected in tests; None composes MetaAgent behind harness_ext
|
agent: MetaAgentCall | None = None # injected in tests; None composes MetaAgent behind harness_ext
|
||||||
|
|
||||||
@@ -190,7 +193,7 @@ class HarnessXProposer:
|
|||||||
self.reason = "harnessx_unavailable"
|
self.reason = "harnessx_unavailable"
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
call = _meta_agent_call()
|
call = _meta_agent_call(self.provider)
|
||||||
except Exception: # noqa: BLE001 — a proposer that cannot start proposes nothing and says so
|
except Exception: # noqa: BLE001 — a proposer that cannot start proposes nothing and says so
|
||||||
self.reason = "harnessx_unavailable"
|
self.reason = "harnessx_unavailable"
|
||||||
return None
|
return None
|
||||||
@@ -263,7 +266,7 @@ def _model_config(model: str, provider: str = "anthropic") -> object:
|
|||||||
return ModelConfig(main=build_provider(model, provider))
|
return ModelConfig(main=build_provider(model, provider))
|
||||||
|
|
||||||
|
|
||||||
def _meta_agent_call() -> MetaAgentCall:
|
def _meta_agent_call(provider: str = "anthropic") -> MetaAgentCall:
|
||||||
"""Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy.
|
"""Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy.
|
||||||
|
|
||||||
`evolve` is a coroutine and returns a HarnessX config path; what this proposer takes from it is the state of the
|
`evolve` is a coroutine and returns a HarnessX config path; what this proposer takes from it is the state of the
|
||||||
@@ -279,7 +282,7 @@ def _meta_agent_call() -> MetaAgentCall:
|
|||||||
trajectories = workspace.parent / "trajectories"
|
trajectories = workspace.parent / "trajectories"
|
||||||
trajectories.mkdir(parents=True, exist_ok=True)
|
trajectories.mkdir(parents=True, exist_ok=True)
|
||||||
(trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8")
|
(trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8")
|
||||||
agent = _meta_agent_class()(inner_model=_model_config(model), allowed_write_roots=(workspace,))
|
agent = _meta_agent_class()(inner_model=_model_config(model, provider), allowed_write_roots=(workspace,))
|
||||||
output = workspace.parent / "meta-out"
|
output = workspace.parent / "meta-out"
|
||||||
output.mkdir(parents=True, exist_ok=True)
|
output.mkdir(parents=True, exist_ok=True)
|
||||||
asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output))
|
asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output))
|
||||||
|
|||||||
@@ -418,7 +418,18 @@ def run_learning_round(
|
|||||||
proposal = proposer.propose(facts) # type: ignore[attr-defined]
|
proposal = proposer.propose(facts) # type: ignore[attr-defined]
|
||||||
record = _round_record(round_number, family, proposal, taxonomy, config)
|
record = _round_record(round_number, family, proposal, taxonomy, config)
|
||||||
if proposal is None:
|
if proposal is None:
|
||||||
return _finish(journal, directory, record, outcome="rejected", reason="no_proposal", cost=spent, trajectories=trajectories)
|
# `no_proposal` covers two different worlds: a proposer that considered the facts and declined, and one
|
||||||
|
# that could not start at all. Losing that distinction hides a fixable configuration error as a quiet round.
|
||||||
|
why = str(getattr(proposer, "reason", "") or "")
|
||||||
|
return _finish(
|
||||||
|
journal,
|
||||||
|
directory,
|
||||||
|
record,
|
||||||
|
outcome="rejected",
|
||||||
|
reason=f"no_proposal: {why}" if why else "no_proposal",
|
||||||
|
cost=spent,
|
||||||
|
trajectories=trajectories,
|
||||||
|
)
|
||||||
refusal = refuse_proposal(proposal, config)
|
refusal = refuse_proposal(proposal, config)
|
||||||
if refusal is not None:
|
if refusal is not None:
|
||||||
return _finish(
|
return _finish(
|
||||||
|
|||||||
@@ -161,7 +161,9 @@ def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace:
|
|||||||
assert len(rounds) == 1 and rounds[0]["family"] == "injection"
|
assert len(rounds) == 1 and rounds[0]["family"] == "injection"
|
||||||
# Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of
|
# Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of
|
||||||
# the enum would have hidden that the round could never propose anything at all.
|
# the enum would have hidden that the round could never propose anything at all.
|
||||||
assert (rounds[0]["outcome"], rounds[0]["reason"]) == ("rejected", "no_proposal")
|
# The reason now carries *why* the proposer had nothing: "no proposal" and "could not start" are different
|
||||||
|
# worlds, and only one of them is a fixable configuration error.
|
||||||
|
assert rounds[0]["outcome"] == "rejected" and rounds[0]["reason"].startswith("no_proposal")
|
||||||
assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file()
|
assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -187,8 +187,13 @@ def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_
|
|||||||
"taxonomy_version": "1",
|
"taxonomy_version": "1",
|
||||||
"metrics": {
|
"metrics": {
|
||||||
"injection": {
|
"injection": {
|
||||||
"family": "injection", "scorable": 10, "unscorable": {}, "recall": recall,
|
"family": "injection",
|
||||||
"silence": 1.0, "youden": recall, "hierarchical_credit": False,
|
"scorable": 10,
|
||||||
|
"unscorable": {},
|
||||||
|
"recall": recall,
|
||||||
|
"silence": 1.0,
|
||||||
|
"youden": recall,
|
||||||
|
"hierarchical_credit": False,
|
||||||
"outcomes": {"pair_correct": 5},
|
"outcomes": {"pair_correct": 5},
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -68,12 +68,12 @@ def test_the_meta_agent_is_built_with_a_model_config_not_a_model_name(tmp_path:
|
|||||||
return workspace
|
return workspace
|
||||||
|
|
||||||
monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent)
|
monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent)
|
||||||
monkeypatch.setattr(proposer_module, "_model_config", lambda model: f"ModelConfig({model})")
|
monkeypatch.setattr(proposer_module, "_model_config", lambda model, provider="anthropic": f"ModelConfig({model}@{provider})")
|
||||||
(tmp_path / "injection").mkdir()
|
(tmp_path / "injection").mkdir()
|
||||||
(tmp_path / "injection" / "checklist.md").write_text("- old\n")
|
(tmp_path / "injection" / "checklist.md").write_text("- old\n")
|
||||||
proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts())
|
proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts())
|
||||||
assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"}
|
assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"}
|
||||||
assert seen["inner_model"] == "ModelConfig(m)", "the agent was handed a model name where it wanted a config"
|
assert seen["inner_model"] == "ModelConfig(m@anthropic)", "the agent was handed a model name where it wanted a config"
|
||||||
assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root"
|
assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -373,3 +373,36 @@ def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None:
|
|||||||
out_dir=tmp_path / "out",
|
out_dir=tmp_path / "out",
|
||||||
k_runs=1,
|
k_runs=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: # type: ignore[no-untyped-def]
|
||||||
|
"""`no_proposal` covers two different worlds: a proposer with nothing to add, and one that could not start.
|
||||||
|
|
||||||
|
Measured: the meta-agent round journalled `no_proposal` while the real cause was an authentication failure in a
|
||||||
|
provider seam pointing at the wrong vendor. A round that failed for a fixable configuration reason must not
|
||||||
|
look identical to a model that considered the facts and declined."""
|
||||||
|
|
||||||
|
class Mute:
|
||||||
|
reason = "meta_agent_failed: RuntimeError"
|
||||||
|
|
||||||
|
def propose(self, facts): # type: ignore[no-untyped-def]
|
||||||
|
del facts
|
||||||
|
return None
|
||||||
|
|
||||||
|
from openultrasast.learning.rounds import run_learning_round
|
||||||
|
|
||||||
|
record = run_learning_round(
|
||||||
|
world["cases"],
|
||||||
|
family="injection",
|
||||||
|
taxonomy=load_families(),
|
||||||
|
configs_dir=world["configs"],
|
||||||
|
proposer=Mute(),
|
||||||
|
scan_factory=_injection_improves,
|
||||||
|
model="m",
|
||||||
|
journal=world["journal"],
|
||||||
|
archive=world["archive"],
|
||||||
|
floors=world["floors"],
|
||||||
|
out_dir=world["out"],
|
||||||
|
)
|
||||||
|
assert record.outcome == "rejected"
|
||||||
|
assert record.reason == "no_proposal: meta_agent_failed: RuntimeError"
|
||||||
|
|||||||
Reference in New Issue
Block a user