feat(learning-harness): family detectors, verifiers and tags in a scan

Task 3.4. A standard scan now classifies each hotspot region and runs the
detectors of every admitted family plus the generalist, reading the
configurations round zero wrote. A scan never creates them: a missing directory
is a recorded reason and the stage is skipped, which is also what happens when
no chat endpoint resolves.

Every claim passes through its family's verifier and carries what produced it:
the family, the detector version and what the verifier said. Nothing rises above
suspicion here, because the verifier writes a tag and the evidence ladder
belongs to another boundary. Reports gain a section naming the three, SARIF
gains the matching properties, and the manifest gains a learning block with the
taxonomy version, the per-family counts, the classifier report and the round.

The feature is off unless a project turns it on, so an existing scan is
unchanged, and all three gate outputs stay byte-identical.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_learning_scan.py

@@ -0,0 +1,105 @@
+"""learning-harness task 3.4: family detectors, verifiers and tags in a scan (offline)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.cli import main
+
+APP = (
+    "from flask import request\n\n\n"
+    "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
+    "    return Book.query.filter_by(book_title=title).first()\n\n\n"
+    "@app.route('/books/any/<title>')\ndef leaky(title):\n"
+    "    owner = request.args.get('owner')\n"
+    "    return Book.query.filter_by(user_id=owner, book_title=title).first()\n"
+)
+
+
+def _repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, toml: str, *, configs: bool = True) -> Path:
+    from openultrasast.learning.detectors import write_default_configs
+    from openultrasast.learning.families import load_families
+
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text(APP)
+    (repo / "openultrasast.toml").write_text(toml)
+    if configs:  # round zero writes these; a scan reads them and never creates them
+        write_default_configs(repo / ".openultrasast/learning/configs", load_families(), prompt="You hunt.", version="0")
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+    monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
+    monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted")
+    return repo
+
+
+def _run_dir(repo: Path) -> Path:
+    return sorted((repo / ".runs").iterdir())[-1]
+
+
+def _scan(repo: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
+    assert main(["scan", str(repo), "--mode", "standard", "--config", str(repo / "openultrasast.toml")]) == 0
+    run_dir = _run_dir(repo)
+    findings = json.loads((run_dir / "findings.json").read_text())["findings"]
+    manifest = json.loads((run_dir / "manifest.json").read_text())
+    return findings, manifest
+
+
+def test_a_standard_scan_tags_findings_with_family_detector_and_verifier(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n')
+    findings, manifest = _scan(repo)
+    detected = [item for item in findings if any(str(tag).startswith("family:") for tag in item["tags"])]
+    assert detected, "the family detectors ran and their findings are in the report"
+    tags = set(detected[0]["tags"])
+    assert any(tag.startswith("family:") for tag in tags)
+    assert any(tag.startswith("detector:") for tag in tags)
+    assert any(tag.startswith("verifier:") for tag in tags)
+    assert detected[0]["evidence_level"] == "suspicion"  # nothing rises without a verifier saying so
+    block = manifest["learning"]
+    assert block["taxonomy_version"] and block["round"] is None
+    assert block["families"] and isinstance(block["classifier"], dict)
+    assert block["classifier"]["hierarchical_credit"] is False
+
+
+def test_a_missing_family_directory_is_a_recorded_reason_not_a_crash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(
+        tmp_path,
+        monkeypatch,
+        '[learning]\nenabled = true\nconfigs_dir = "nowhere/configs"\n\n[models]\nhunter = "scripted-model"\n',
+        configs=False,
+    )
+    findings, manifest = _scan(repo)
+    assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"])
+    reasons = {str(item.get("reason")) for item in manifest["degradations"]}
+    assert "learning_configs_unavailable" in reasons
+    assert "learning" not in manifest or manifest["learning"]["families"] == {}
+
+
+def test_the_feature_is_off_by_configuration_and_leaves_no_trace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(tmp_path, monkeypatch, "[learning]\nenabled = false\n", configs=False)
+    findings, manifest = _scan(repo)
+    assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"])
+    assert "learning" not in manifest
+
+
+def test_family_findings_never_become_sandbox_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n')
+    monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
+    monkeypatch.setenv("OPENULTRASAST_SANDBOX_RUNNER", "fake")
+    assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0
+    run_dir = _run_dir(repo)
+    verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
+    findings = {item["finding_id"] for item in json.loads((run_dir / "findings.json").read_text())["findings"]}
+    del findings
+    assert all(not str(item.get("path", "")).startswith("family:") for item in verdicts)
+
+
+def test_the_sarif_report_carries_the_family_properties(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n')
+    _scan(repo)
+    sarif = json.loads((_run_dir(repo) / "report.sarif").read_text())
+    props = [result["properties"] for result in sarif["runs"][0]["results"]]
+    family_props = [item for item in props if item.get("family")]
+    assert family_props and all(item.get("detector") and item.get("verifier") for item in family_props)