Open_Ultra_SAST/tests/test_learn_schema.py

"""The candidate feature schema (learned-decision-engine task 1; Req 1.2, 7.2): a closed allow-list with no
identities, missing distinct from zero, and no label-carrying field reaching a feature."""

from __future__ import annotations

import copy
import tomllib
from pathlib import Path
from typing import Any

import pytest

from openultrasast.findings import quick_scan_findings
from openultrasast.learn import schema
from openultrasast.learn.features import Delta, Part, build_for_scan, record
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
from openultrasast.plane.tasks.features import case_features
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets

ROOT = Path(__file__).resolve().parents[1]
APP = """import os


def {inner}(value):
    os.system(value)


def {outer}():
    {inner}("x")
"""


def _scan(root: Path) -> list[dict[str, Any]]:
    _, targets = preprocess_repository(root)
    assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read"
    targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
    findings = quick_scan_findings(root, targets, rank_targets(targets))
    assert findings, "the fixture's os.system fires a quick rule"
    engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []}
    return build_for_scan(root, findings, engine)


def _repo(root: Path, path: str, inner: str, outer: str) -> Path:
    (root / path).parent.mkdir(parents=True, exist_ok=True)
    (root / path).write_text(APP.format(inner=inner, outer=outer))
    return root


# --- the allow-list ----------------------------------------------------------------------------------------------


def test_features_are_allow_listed(tmp_path: Path) -> None:
    names = [spec.name for spec in FEATURES]
    assert len(names) == len(set(names)), "a feature name is declared twice"
    for spec in FEATURES:
        assert spec.type in ("bool", "int", "float", "enum"), f"{spec.name}: no free text"
        assert spec.instrument in schema.INSTRUMENTS and spec.profile in schema.PROFILES
        assert (spec.type == "int") == (spec.cap is not None), spec.name
        assert (spec.type == "enum") == bool(spec.vocabulary), spec.name
        assert spec.name.rsplit(".", 1)[-1] not in EXCLUDED_FIELDS, f"{spec.name} names a label or identity field"
    mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
    assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
    assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
    records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta"))
    assert records
    for rec in records:
        assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES"
        assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"])
    assert {s.name for s in schema.features_for("static")} < set(names)
    assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]


def test_record_is_name_invariant(tmp_path: Path) -> None:
    """Two repositories that differ only in repository name, paths, file and function names and commit ids give the
    same features: nothing identifies a repository."""
    first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta"))
    second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta"))
    assert [r["x"] for r in first] == [r["x"] for r in second]
    assert [r["instruments"] for r in first] == [r["instruments"] for r in second]
    assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope"
    assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None
    root = tmp_path / "acme-shop"
    _, targets = preprocess_repository(root)
    findings = quick_scan_findings(root, targets, rank_targets(targets))
    changed = {"app/one.py": [(5, 5)]}
    one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40))
    two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40))
    assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40
    assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1
    assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature"


# --- label fields ------------------------------------------------------------------------------------------------

AGREED = {
    "family": "injection",
    "candidates": [
        {"candidate": "a.py::f", "site": "a.py:4:f", "a": True, "b": False, "c": True, "agreed": True, "site_match": True,
         "finding": {"family": "injection", "reported_at": "a.py:5"}},
    ],
    "disputed": [],
}  # fmt: skip
PASSES = {
    "a": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [{"candidate": "a.py::f", "reported_at": "a.py:5"}], "turns": 4}],
    "b": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [], "turns": 6}],
    "c": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [{"candidate": "a.py::f", "reported_at": "a.py:4"}], "turns": 5}],
}
ALERTS = [
    {"rule_id": "python-os-command", "rule_status": "enabled", "path": "a.py", "line": 5, "function": "f", "pin_role": "vulnerable",
     "in_fix_range": True},
]  # fmt: skip
SUMMARY = {"status": "done", "quick_languages": ["python"], "engine_languages": ["php"]}


def _plane(**overrides: Any) -> list[dict[str, Any]]:
    inputs: dict[str, Any] = {
        "facts": {"callers": {"a.py::f": [{"path": "b.py", "line": 2}]}}, "passes": PASSES, "models": {"a": "m", "b": "m", "c": "m"},
        "agreed": AGREED, "alerts": ALERTS, "alerts_summary": SUMMARY,
    }  # fmt: skip
    return case_features(**{**inputs, **overrides})


def test_label_fields_never_reach_features() -> None:
    baseline = _plane()
    assert [r["candidate"] for r in baseline] == ["a.py::f"] and baseline[0]["profile"] == "plane"
    x = baseline[0]["x"]
    assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, False, True, 2)
    assert x["agree.final"] == "agreed" and x["qr.enabled_hits"] == 1 and x["verify.site_offset"] == 0
    flipped = copy.deepcopy(AGREED)
    flipped["candidates"][0]["site_match"] = False
    outside = [{**ALERTS[0], "in_fix_range": False}]
    fixed_pin = [*ALERTS, {**ALERTS[0], "pin_role": "fixed", "line": 9}, {**ALERTS[0], "pin_role": "fixed", "function": "g"}]
    for variant in (_plane(agreed=flipped), _plane(alerts=outside), _plane(alerts=fixed_pin)):
        assert variant == baseline, "site_match, in_fix_range and fixed-pin alerts change nothing"
    for field in ("site_match", "in_fix_range", "rule_id", "path", "repo", "usd"):
        smuggled = copy.deepcopy(baseline[0])
        smuggled["x"][field] = True
        with pytest.raises(FeatureRecordError, match="allow-list"):
            validate_record(smuggled)
        with pytest.raises(FeatureRecordError):
            validate_record({**baseline[0], field: True})


# --- missing is not zero -----------------------------------------------------------------------------------------


def test_missing_is_not_zero() -> None:
    uncovered = _plane(alerts=[], alerts_summary={**SUMMARY, "quick_languages": ["php"]})[0]
    assert uncovered["instruments"]["quick"]["state"] == "none"
    assert all(v is None for k, v in uncovered["x"].items() if k.startswith("qr."))
    covered = _plane(alerts=[])[0]
    assert covered["instruments"]["quick"]["state"] == "ran"
    assert covered["x"]["qr.enabled_hits"] == 0 and covered["x"]["qr.shadow_hits"] == 0, "ran and found nothing is 0"
    no_alerts = _plane(alerts=[], alerts_summary=None)[0]
    assert no_alerts["instruments"]["quick"]["state"] == "none" and no_alerts["x"]["qr.enabled_hits"] is None
    assert covered["instruments"]["engine"]["state"] == "none" and covered["x"]["eng.findings"] is None
    assert covered["instruments"]["delta"]["state"] == "not_applicable" and covered["x"]["delta.novelty"] is None
    failed = _plane(alerts_summary={**SUMMARY, "status": "failed"})[0]
    assert failed["instruments"]["quick"]["state"] == "failed" and failed["x"]["qr.enabled_hits"] is None
    php = {"path": "p.php", "line": 3, "function": "q", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled"}
    engine = {**SUMMARY, "engine": {"vulnerable": {"questions": 0, "completed": 0, "files": 5, "degradations": []}}}
    [_, unread] = _plane(alerts=[*ALERTS, php], alerts_summary=engine)
    assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"


def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None:
    root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta")
    _, targets = preprocess_repository(root)
    findings = quick_scan_findings(root, targets, rank_targets(targets))
    unread = build_for_scan(root, findings, {"questions": 0, "findings": []})
    absent = build_for_scan(root, findings, None)
    assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"}
    assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"}
    assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent))


def test_validate_record_refuses_values_outside_their_type() -> None:
    good = _plane()[0]
    validate_record(good)
    for name, value in (("agree.final", "maybe"), ("verify.votes", 4), ("qr.enabled_hits", -1), ("lang.python", 1)):
        bad = copy.deepcopy(good)
        bad["x"][name] = value
        with pytest.raises(FeatureRecordError, match=name.replace(".", r"\.")):
            validate_record(bad)
    null = copy.deepcopy(good)
    null["x"]["agree.final"] = None
    with pytest.raises(FeatureRecordError, match="null is not allowed"):
        validate_record(null)
    value = copy.deepcopy(good)
    value["x"]["ms.flagged"] = False
    with pytest.raises(FeatureRecordError, match="must be null"):
        validate_record(value)
    with pytest.raises(FeatureRecordError, match="outside the static profile"):
        record("a.py::f", "injection", "python", "static", {"verify": Part("ran", {})})