feat(authorization-obligations): leave-one-out per obligation kind

Task 4.2. For an obligation-labeled pair, leave-one-out runs the checker on both
sides with the held-out store's obligation shapes. The pair counts as detected
only when a finding inside the labeled function carries both the labeled
obligation and a known fix from a store record, so the checker's own facts-only
finding is never mistaken for the corpus teaching. Recall, silence and Youden
are reported per obligation kind beside the flow mechanisms.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_loo.py

@@ -156,3 +156,76 @@ def test_leave_one_out_forwards_search_degradations(tmp_path: Path, monkeypatch:
     )
     result = evaluate_loo([*_toy(tmp_path), case])
     assert any(item["reason"] == "variants_language_unsupported" for item in result.degradations)
+
+
+# --- authorization-obligations 4.2: leave-one-out per obligation kind -------------------------------------------------
+
+_HANDLER = "from flask import request\n\n\n@app.route('/{route}/<key>')\ndef {fn}(key):\n{body}"
+OBL_A = _HANDLER.format(route="books", fn="get_book", body="    return Book.query.filter_by(book_title=key).first()\n")
+OBL_A_FIX = _HANDLER.format(
+    route="books",
+    fn="get_book",
+    body="    user_id = request.user.id\n    return Book.query.filter_by(user_id=user_id, book_title=key).first()\n",
+)
+OBL_B = _HANDLER.format(route="orders", fn="get_order", body="    return Order.query.filter_by(ref=key).first()\n")
+OBL_B_FIX = _HANDLER.format(
+    route="orders", fn="get_order", body="    owner = current_user.id\n    return Order.query.filter_by(owner=owner, ref=key).first()\n"
+)
+OBL_C = _HANDLER.format(
+    route="login", fn="login", body="    resp = make_response('ok')\n    resp.set_cookie('sid', key, samesite='None')\n    return resp\n"
+)
+OBL_C_FIX = _HANDLER.format(
+    route="login",
+    fn="login",
+    body="    resp = make_response('ok')\n    resp.set_cookie('sid', key, samesite='Strict', httponly=1)\n    return resp\n",
+)
+
+
+def _obligation_case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, obligation: str, mechanism: str) -> PairCase:
+    case = _case(tmp_path, name, vuln, fixed, function=function, cwe="CWE-639")
+    row = case.expected[0]
+    return PairCase(
+        **{
+            **case.__dict__,
+            "slice": "vibe-py",
+            "expected": (ExpectedFinding(**{**row.__dict__, "mechanism": mechanism, "obligation": obligation}),),
+        }
+    )
+
+
+def _obligation_toy(tmp_path: Path) -> list[PairCase]:
+    read = {"obligation": "protected_read", "mechanism": "unconstrained_protected_read"}
+    return [
+        _obligation_case(tmp_path, "a", OBL_A, OBL_A_FIX, function="get_book", **read),
+        _obligation_case(tmp_path, "b", OBL_B, OBL_B_FIX, function="get_order", **read),
+        _obligation_case(tmp_path, "c", OBL_C, OBL_C_FIX, function="login", obligation="security_setting", mechanism="permissive_default"),
+    ]
+
+
+def test_leave_one_out_scores_obligation_rows_per_kind_and_names_the_teaching_pair(tmp_path: Path) -> None:
+    """Req 7.4: recall, silence and Youden per obligation kind; found_by names the shape record that supplied known_fix."""
+    from openultrasast.semantic.loo import evaluate_loo
+
+    result = evaluate_loo(_obligation_toy(tmp_path))
+    by_name = {outcome.pair: outcome for outcome in result.outcomes}
+    a, b, c = by_name["a"], by_name["b"], by_name["c"]
+    assert a.obligations == ("protected_read",) and c.obligations == ("security_setting",)
+    assert a.detected and a.silent and [pairs for _, pairs in a.found_by] == [("b",)]  # b taught the identity discharger
+    assert b.detected and b.silent and [pairs for _, pairs in b.found_by] == [("a",)]
+    assert all(record_id.startswith("corpus:") for record_id, _ in a.found_by)
+    assert c.teaches and not c.detected and c.found_by == () and c.silent  # reached and undischarged, but nobody else teaches the kind
+    kinds = result.per_obligation_kind
+    assert kinds["protected_read"]["pairs"] == 2 and kinds["protected_read"]["detected"] == 2 and kinds["protected_read"]["youden"] == 1.0
+    assert kinds["security_setting"]["pairs"] == 1 and kinds["security_setting"]["detected"] == 0
+    assert "unconstrained_protected_read" in result.per_mechanism  # flow mechanisms and obligation kinds sit side by side
+    payload = result.to_dict()
+    assert "per_obligation_kind" in payload and payload["outcomes"][0]["obligations"] == ["protected_read"]
+    json.dumps(payload)
+
+
+def test_obligation_rows_are_not_found_by_the_facts_alone(tmp_path: Path) -> None:
+    """A finding without a known fix from the held-out store is the checker's own work, not the corpus teaching."""
+    from openultrasast.semantic.loo import evaluate_loo
+
+    (only,) = [outcome for outcome in evaluate_loo(_obligation_toy(tmp_path)[:1]).outcomes]
+    assert only.hits_vuln >= 1 and not only.detected and only.found_by == ()  # the leaky read is flagged, but no pair taught it