feat(learning-harness): the classifier measures itself and queues disagreements

Task 2.2. The report covers the whole corpus and scores the classifier only
where a maintainer label exists. Agreement uses the classifier's first answer,
the oracle number uses the best of its answers, and the random baseline is a
uniform pick over the nine real families, so a router error is visible on its
own rather than hidden inside a detector number. Credit is hierarchical: full
for the same family, half for a parent or child, none for a lateral or
fabricated id.

When it is being measured, the classifier no longer reads the declared label it
is scored against, otherwise agreement would be the classifier reading back its
own answer sheet. With no maintainer labels in the corpus yet the report says
the reference set is empty instead of inventing a score, and every disagreement
is queued once with both answers and the deciding tier for a human to settle.

Measured on the 277 vendored pairs: tier one places 275 of them without a single
model call, only two land in unknown, and the 176 memory-safety rows resolve to
one family instead of reading as an unclassified catch-all.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_learning_classifier_report.py

@@ -0,0 +1,101 @@
+"""learning-harness task 2.2: the classifier reports on itself and queues disagreements (offline)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from openultrasast.benchmark import ExpectedFinding
+from openultrasast.learning.families import load_families
+from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored
+
+
+def _case(tmp_path: Path, name: str, *, cwe: str, mechanism: str = "other", family: str | None = None, tier: str = "reviewed") -> PairCase:
+    (tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
+    return PairCase(
+        name=name,
+        slice="vibe-py",
+        language="python",
+        origin="test",
+        vuln_file=tmp_path / f"{name}.py",
+        fixed_file=tmp_path / f"{name}.py",
+        relpath="app.py",
+        expected=(
+            ExpectedFinding(cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function="f", mechanism=mechanism, family=family),
+        ),
+        min_recall=1.0,
+        fix_policy="silent",
+        review_tier=tier,
+        reviewer="m" if tier == "reviewed" else "",
+    )
+
+
+def test_the_report_covers_the_whole_corpus_and_names_the_unknowns(tmp_path: Path) -> None:
+    from openultrasast.learning.classify import measure_classifier
+
+    taxonomy = load_families()
+    cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG))
+    report = measure_classifier(cases, taxonomy, client=None)
+    assert report.taxonomy_version == taxonomy.version
+    assert report.classified == len(cases)
+    assert sum(report.per_tier.values()) == len(cases) and set(report.per_tier) <= {"static", "model", "none"}
+    assert report.per_family["memory"] >= 150  # the vfc rows land in one family instead of reading `other`
+    assert report.per_family["access_control"] >= 10 and report.per_family["injection"] >= 40
+    assert report.unknown == 2 and report.per_tier["static"] == len(cases) - 2  # tier one places all but two rows
+    payload = report.to_dict()
+    json.dumps(payload)
+    assert payload["labeled"] == 0  # no maintainer family labels in the corpus yet, and the report says so
+
+
+def test_agreement_uses_the_first_answer_and_the_oracle_uses_any_of_them(tmp_path: Path) -> None:
+    from openultrasast.learning.classify import measure_classifier
+
+    taxonomy = load_families()
+    cases = [
+        _case(tmp_path, "hit", cwe="CWE-89", family="injection"),  # tier one answers injection: agreement and oracle
+        _case(tmp_path, "miss", cwe="CWE-89", family="access_control"),  # answers injection: neither
+    ]
+    report = measure_classifier(cases, taxonomy, client=None)
+    assert report.labeled == 2
+    assert report.agreement == 0.5 and report.oracle_router == 0.5
+    assert abs(report.random_router - 1 / 9) < 1e-9  # a uniform pick over the nine real families
+    assert report.confusion["injection"]["injection"] == 1 and report.confusion["access_control"]["injection"] == 1
+
+
+def test_every_disagreement_is_queued_once_with_both_answers_and_the_tier(tmp_path: Path) -> None:
+    from openultrasast.learning.classify import measure_classifier, write_review_queue
+
+    taxonomy = load_families()
+    cases = [_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), _case(tmp_path, "hit", cwe="CWE-89", family="injection")]
+    report = measure_classifier(cases, taxonomy, client=None)
+    assert [entry.pair for entry in report.queue] == ["miss"]
+    entry = report.queue[0]
+    assert entry.label == "access_control" and entry.classified == ("injection",) and entry.tier == "static"
+    queue_path = tmp_path / "review-queue.jsonl"
+    write_review_queue(queue_path, report.queue)
+    write_review_queue(queue_path, report.queue)  # a second measurement must not duplicate the row
+    rows = [json.loads(line) for line in queue_path.read_text().splitlines() if line.strip()]
+    assert [row["pair"] for row in rows] == ["miss"]
+    assert rows[0]["label"] == "access_control" and rows[0]["classified"] == ["injection"] and rows[0]["tier"] == "static"
+
+
+def test_measuring_never_writes_a_label(tmp_path: Path) -> None:
+    from openultrasast.learning.classify import measure_classifier
+
+    taxonomy = load_families()
+    case = _case(tmp_path, "miss", cwe="CWE-89", family="access_control")
+    measure_classifier([case], taxonomy, client=None)
+    assert case.expected[0].family == "access_control"  # the classifier proposes; the maintainer decides
+
+
+def test_only_reviewed_and_seeded_rows_count_towards_agreement(tmp_path: Path) -> None:
+    from openultrasast.learning.classify import measure_classifier
+
+    taxonomy = load_families()
+    cases = [
+        _case(tmp_path, "trusted", cwe="CWE-89", family="access_control", tier="reviewed"),
+        _case(tmp_path, "untrusted", cwe="CWE-89", family="access_control", tier="title"),
+    ]
+    report = measure_classifier(cases, taxonomy, client=None)
+    assert report.labeled == 1 and report.classified == 2  # an unreviewed label is not a reference answer
+    assert [entry.pair for entry in report.queue] == ["trusted"]