fix(learning-harness): the three defects the group 2 review found

Task 2.3: two of the five unscorable reasons had no code path, so an unparsable
row was reported as a corpus labelling defect when the true cause was a tooling
gap. They have different owners and different fixes, and conflating them is the
dishonesty this spec exists to remove. The design's unscorable_reason is
reinstated with the parse result as an input, and a handler that arrived without
its registration is now named as missing context for the families whose bug
shape is a missing check.

Task 2.6: a verification stamped its own word for proof onto the shared evidence
level, which is not a value the project's ladder defines, and three existing
helpers raised on it. Nothing in production called it yet, so the suite stayed
green while the wiring in the next group would have broken a scan. A
verification now writes only its tag, and a caller that wants to move a claim up
the ladder goes through an explicit mapping that can only produce a level the
ladder has.

Task 2.7: the injection canary planted its row in a private in-memory database
the target could never reach, so it could not fire; the fake sandbox handed the
token back regardless of the snippet, which is why nothing noticed. It is now an
execution canary whose marker only an interpreter could write, and each template
is tested by running it for real against a vulnerable fixture and its fixed
twin. That test immediately caught a second flaw: a handler that merely echoes
its input printed the oracle token and satisfied the oracle without any
vulnerability, so an injected payload now carries a value unrelated to the token
the oracle looks for.

Also from the review: a configuration with no tools is refused instead of
producing a detector that silently reports nothing, the classifier abstains when
its fallback call fails too, and the archive docstring says which version wins.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_learning_canaries.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import json
 from pathlib import Path
 
+import pytest
+
 from openultrasast.config import SandboxConfig
 from openultrasast.findings import StaticFinding
 from openultrasast.learning.families import load_families
@@ -47,7 +49,7 @@ def test_every_canary_template_renders_and_passes_the_snippet_safety_check() ->
     assert render_canary("injection", language="rust", path="a.rs", function="f") is None
 
 
-def test_a_canary_token_is_unique_per_claim_so_one_family_never_answers_for_another() -> None:
+def test_a_canary_token_is_unique_to_the_experiment_it_names() -> None:
     from openultrasast.learning.canaries import render_canary
 
     first = render_canary("injection", language="python", path="app.py", function="lookup")
@@ -55,6 +57,8 @@ def test_a_canary_token_is_unique_per_claim_so_one_family_never_answers_for_anot
     third = render_canary("path", language="python", path="app.py", function="lookup")
     assert first and second and third
     assert first[1] != second[1] != third[1] and first[1] != third[1]
+    # the same family, file and function is the same experiment, so it is deliberately the same token
+    assert render_canary("injection", language="python", path="app.py", function="lookup")[1] == first[1]  # type: ignore[index]
 
 
 def test_the_canary_verifier_proves_only_when_the_oracle_fires(tmp_path: Path) -> None:
@@ -143,3 +147,51 @@ def _runner(reply: object) -> FakeSandboxRunner:
             return reply(marker)  # type: ignore[operator]
 
     return _Runner()
+
+
+VULNERABLE = {
+    "injection": "import os\n\n\ndef lookup(value):\n    os.system(value)\n    return 'done'\n",
+    "path": "def lookup(value):\n    with open(value, encoding='utf-8') as handle:\n        return handle.read()\n",
+    "deserialization": "import pickle\n\n\ndef lookup(value):\n    return pickle.loads(value)\n",
+}
+FIXED = {
+    "injection": (
+        "import shlex, subprocess\n\n\ndef lookup(value):\n"
+        "    subprocess.run(['echo', shlex.quote(value)], check=False)\n    return 'done'\n"
+    ),
+    "path": (
+        "from pathlib import Path\n\n\ndef lookup(value):\n"
+        "    name = Path(value).name\n    return (Path('/tmp') / name).read_text(encoding='utf-8') if False else ''\n"
+    ),
+    "deserialization": "import json\n\n\ndef lookup(value):\n    return json.loads(value.decode('utf-8', 'ignore') or '{}')\n",
+}
+
+
+@pytest.mark.parametrize("family", CANARY_FAMILIES)
+def test_each_canary_fires_on_a_real_bug_and_stays_quiet_on_its_fixed_twin(tmp_path: Path, family: str) -> None:
+    """The oracle must be a property of the code, not of the harness: run the snippet for real, both sides.
+
+    The injection fixture's fixed twin deliberately echoes its input, which is why the payload carries a
+    value unrelated to the oracle token: an echoing handler must not be able to satisfy the oracle.
+    """
+    import subprocess
+    import sys
+
+    from openultrasast.learning.canaries import render_canary
+
+    for side, sources in (("vuln", VULNERABLE), ("fixed", FIXED)):
+        workspace = tmp_path / side / "workspace"
+        scratch = tmp_path / side / "scratch"
+        workspace.mkdir(parents=True)
+        scratch.mkdir(parents=True)
+        (workspace / "app.py").write_text(sources[family])
+        rendered = render_canary(
+            family, language="python", path="app.py", function="lookup", workspace=str(workspace), scratch=str(scratch)
+        )
+        assert rendered is not None
+        snippet, token = rendered
+        script = scratch / "case.py"
+        script.write_text(snippet)
+        completed = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, timeout=60, check=False)
+        fired = token in completed.stdout or token in completed.stderr
+        assert fired is (side == "vuln"), f"{family} {side}: {completed.stdout!r} {completed.stderr!r}"

tests/test_learning_detectors.py

@@ -156,3 +156,12 @@ def test_round_zero_can_write_a_configuration_for_every_family(tmp_path: Path) -
     assert all(config.version == "0" for config in configs.values())
     assert all(config.prompt.startswith("You are a security hunter.") for config in configs.values())
     assert configs["access_control"].prompt != configs["injection"].prompt  # each carries its own family's definition
+
+
+def test_a_configuration_with_no_tools_is_refused_rather_than_reporting_nothing(tmp_path: Path) -> None:
+    """A detector offered no tools never calls one, and the hunter loop then returns nothing at all."""
+    from openultrasast.learning.detectors import DetectorConfigError, load_family_configs
+
+    _write_config(tmp_path, "injection", tools=[])
+    with pytest.raises(DetectorConfigError, match="tools"):
+        load_family_configs(tmp_path, load_families())

tests/test_learning_scoring.py

@@ -150,3 +150,36 @@ def test_the_rung_is_the_best_verifier_outcome_among_the_detections(tmp_path: Pa
     assert score.outcome == "pair_correct" and score.rung == "static_corroboration"
     plain = score_pair_family(_case(tmp_path), "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
     assert plain.rung == "suspicion"
+
+
+def test_the_unscorable_reason_tells_a_tooling_gap_from_a_corpus_defect(tmp_path: Path) -> None:
+    """Req 4.1: an unparsable row is a tooling gap; a row whose labeled function no range names is a corpus defect."""
+    from openultrasast.learning.scoring import unscorable_reason
+
+    case = _case(tmp_path)
+    assert unscorable_reason(case, parse_ok=True, ranges=RANGES) is None
+    assert unscorable_reason(case, parse_ok=False, ranges={}) == "unsupported_language"
+    assert unscorable_reason(case, parse_ok=False, ranges=RANGES) == "unsupported_language"  # the parse decides, not the ranges
+    assert unscorable_reason(case, parse_ok=True, ranges={}) == "unresolved_label"
+    assert unscorable_reason(_case(tmp_path, unscorable="identical_twin"), parse_ok=True, ranges=RANGES) == "identical_twin"
+    twin = _case(tmp_path, unscorable="known_limit:java hash")
+    assert unscorable_reason(twin, parse_ok=False, ranges={}) == "known_limit:java hash"  # a declared limit wins
+
+
+def test_a_row_whose_registration_context_is_missing_is_named_as_such(tmp_path: Path) -> None:
+    """Req 4.1: an access-control row whose handler arrived without its registration cannot be scored for that reason."""
+    from openultrasast.learning.scoring import unscorable_reason
+
+    case = _case(tmp_path, family="access_control")
+    assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=("leaky",)) is None
+    assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=()) == "missing_context"
+    # only families whose bug shape needs the registration are judged on it
+    assert unscorable_reason(_case(tmp_path, family="injection"), parse_ok=True, ranges=RANGES, entry_points=()) is None
+
+
+def test_an_unparsable_side_is_scored_unscorable_rather_than_missed(tmp_path: Path) -> None:
+    from openultrasast.learning.scoring import score_pair_family
+
+    taxonomy = load_families()
+    score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy, parse_ok=False)
+    assert score.outcome == "unscorable" and score.unscorable_reason == "unsupported_language"

tests/test_learning_verifiers.py

@@ -103,13 +103,43 @@ def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_w
     assert elsewhere.rung == "suspicion"
 
 
-def test_a_verification_can_be_written_onto_the_claim_as_a_tag(repo: Path) -> None:
+def test_a_verification_is_written_onto_the_claim_as_a_tag_and_nothing_else(repo: Path) -> None:
+    """The evidence ladder belongs to another boundary; a verifier says what it saw with a tag."""
     from openultrasast.learning.verifiers import apply_verification, verifier_for
 
     taxonomy = load_families()
     claim = _claim("leaky", 12, "access_control")
     verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python"))
     assert "verifier:static_corroboration" in verified.tags
-    assert verified.evidence_level == "static_corroboration" and claim.evidence_level == "suspicion"
+    assert verified.evidence_level == "suspicion" and claim.evidence_level == "suspicion"  # the ladder is untouched
     plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python"))
-    assert "verifier:suspicion" in plain.tags and plain.evidence_level == "suspicion"
+    assert "verifier:suspicion" in plain.tags and plain.tags.count("verifier:suspicion") == 1
+
+
+def test_a_rung_only_reaches_the_ladder_through_an_explicit_mapping() -> None:
+    """Every rung must land on a level the project's ladder actually defines, or the report raises later."""
+    from openultrasast.learning.verifiers import RUNGS, evidence_level_for
+    from openultrasast.verification import EvidenceLevel
+
+    assert {rung: evidence_level_for(rung) for rung in RUNGS} == {
+        "suspicion": EvidenceLevel.SUSPICION,
+        "static_corroboration": EvidenceLevel.STATIC_CORROBORATION,
+        "proven": EvidenceLevel.EXPLOIT_DEMONSTRATED,
+    }
+    assert all(EvidenceLevel(evidence_level_for(rung).value) for rung in RUNGS)  # never a value outside the ladder
+
+
+def test_a_verified_claim_still_passes_the_projects_own_verification_helpers(repo: Path) -> None:
+    """The bug this pins: stamping a learning rung onto evidence_level raised ValueError in three call sites."""
+    from openultrasast.learning.verifiers import Verification, apply_verification, raise_to
+    from openultrasast.verification import VerificationStatus, is_report_verified, verify_finding
+
+    claim = _claim("lookup", 3, "injection")
+    proven = apply_verification(claim, Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY x"))
+    assert "verifier:proven" in proven.tags
+    assert verify_finding(proven).evidence_level == "suspicion"  # would raise ValueError if the rung were stamped on
+    assert is_report_verified(proven.evidence_level, VerificationStatus.ACCEPTED) is False
+    raised = raise_to(proven, "proven")
+    assert raised.evidence_level == "exploit_demonstrated"
+    assert verify_finding(raised).evidence_level == "exploit_demonstrated"
+    assert is_report_verified(raised.evidence_level, VerificationStatus.ACCEPTED) is True