"""learning-harness task 3.4: family detectors, verifiers and tags in a scan (offline).""" from __future__ import annotations import json from pathlib import Path import pytest from openultrasast.cli import main APP = ( "from flask import request\n\n\n" "@app.route('/books/')\n@login_required\ndef guarded(title):\n" " return Book.query.filter_by(book_title=title).first()\n\n\n" "@app.route('/books/any/<title>')\ndef leaky(title):\n" " owner = request.args.get('owner')\n" " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" ) def _repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, toml: str, *, configs: bool = True) -> Path: from openultrasast.learning.detectors import write_default_configs from openultrasast.learning.families import load_families repo = tmp_path / "repo" repo.mkdir() (repo / "app.py").write_text(APP) (repo / "openultrasast.toml").write_text(toml) if configs: # round zero writes these; a scan reads them and never creates them write_default_configs(repo / ".openultrasast/learning/configs", load_families(), prompt="You hunt.", version="0") monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") return repo def _run_dir(repo: Path) -> Path: return sorted((repo / ".runs").iterdir())[-1] def _scan(repo: Path) -> tuple[list[dict[str, object]], dict[str, object]]: assert main(["scan", str(repo), "--mode", "standard", "--config", str(repo / "openultrasast.toml")]) == 0 run_dir = _run_dir(repo) findings = json.loads((run_dir / "findings.json").read_text())["findings"] manifest = json.loads((run_dir / "manifest.json").read_text()) return findings, manifest def test_a_standard_scan_tags_findings_with_family_detector_and_verifier(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') findings, manifest = _scan(repo) detected = [item for item in findings if any(str(tag).startswith("family:") for tag in item["tags"])] assert detected, "the family detectors ran and their findings are in the report" tags = set(detected[0]["tags"]) assert any(tag.startswith("family:") for tag in tags) assert any(tag.startswith("detector:") for tag in tags) assert any(tag.startswith("verifier:") for tag in tags) assert detected[0]["evidence_level"] == "suspicion" # nothing rises without a verifier saying so block = manifest["learning"] assert block["taxonomy_version"] and block["round"] is None assert block["families"] and isinstance(block["classifier"], dict) assert block["classifier"]["hierarchical_credit"] is False def test_a_missing_family_directory_is_a_recorded_reason_not_a_crash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo = _repo( tmp_path, monkeypatch, '[learning]\nenabled = true\nconfigs_dir = "nowhere/configs"\n\n[models]\nhunter = "scripted-model"\n', configs=False, ) findings, manifest = _scan(repo) assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"]) reasons = {str(item.get("reason")) for item in manifest["degradations"]} assert "learning_configs_unavailable" in reasons assert "learning" not in manifest or manifest["learning"]["families"] == {} def test_the_feature_is_off_by_configuration_and_leaves_no_trace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo = _repo(tmp_path, monkeypatch, "[learning]\nenabled = false\n", configs=False) findings, manifest = _scan(repo) assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"]) assert "learning" not in manifest def test_family_findings_never_become_sandbox_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") monkeypatch.setenv("OPENULTRASAST_SANDBOX_RUNNER", "fake") assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0 run_dir = _run_dir(repo) verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] findings = {item["finding_id"] for item in json.loads((run_dir / "findings.json").read_text())["findings"]} del findings assert all(not str(item.get("path", "")).startswith("family:") for item in verdicts) def test_the_sarif_report_carries_the_family_properties(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') _scan(repo) sarif = json.loads((_run_dir(repo) / "report.sarif").read_text()) props = [result["properties"] for result in sarif["runs"][0]["results"]] family_props = [item for item in props if item.get("family")] assert family_props and all(item.get("detector") and item.get("verifier") for item in family_props)