Open_Ultra_SAST/tests/test_unseen_calibrate.py

"""Offline calibration, placement, agreement and Docker diagnostic controls."""

import importlib
import json
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
c = importlib.import_module("benchmarks.unseen.calibrate")
r = importlib.import_module("benchmarks.unseen.replay")
s = importlib.import_module("benchmarks.unseen.score")
b = importlib.import_module("benchmarks.ax.batch")


def timing_record(identity="opaque", factor=1, lane="docker"):
    return {
        "id": identity,
        "lane": lane,
        "repo": "never-publish",
        "kind": "ordinary",
        "instrument": {
            "head_verified": True,
            "base_verified": True,
            "changed_bytes": {"app.py": 10},
            "hook_exit": 0,
            "hook_bytes_read": 10,
            "timings": {"pre-push": 60 * factor},
        },
        "push": {
            "timings": {
                f"comparison_0_{k}_seconds": v * factor
                for k, v in (("resolution", 1), ("provenance", 2), ("snapshot", 4), ("preparation", 6), ("quick", 1))
            },
            "quick_tier": [{"bytes_read": 10, "findings": []}],
            "scans": [{"scan": {"build_seconds": 20 * factor, "query_seconds": 30 * factor}}],
        },
    }


def test_calibration_median_bootstrap_and_privacy():
    result = c.slowdown([1, 10, 100], [2, 20, 200])
    assert result == {"n": 3, "factor": 2, "bootstrap95": [2, 2]}
    varied = c.slowdown([1] * 5, [1, 2, 3, 4, 5])
    assert varied["factor"] == 3
    assert varied["bootstrap95"] == [1, 5]
    assert varied == c.slowdown([1] * 5, [1, 2, 3, 4, 5])
    record = c.calibration_record([timing_record()], [timing_record(factor=3, lane="ax")])
    assert record["quick_total"]["factor"] == 3
    assert record["pairs"][0]["native"]["quick_total"] == 10  # Snapshot is nested, not added twice.
    assert record["engine_build"]["factor"] == record["engine_queries"]["factor"] == 3
    assert "never-publish" not in json.dumps(record)
    with pytest.raises(ValueError):
        c.slowdown([0], [2])
    bad = timing_record()
    bad["push"]["snapshots"] = [{"boundaries": [{"reason": "deadline_exhausted"}]}]
    with pytest.raises(ValueError, match="deadline"):
        c.stage_times(bad)
    bad = timing_record()
    del bad["push"]["timings"]["comparison_0_quick_seconds"]
    with pytest.raises(ValueError, match="missing"):
        c.stage_times(bad)


@pytest.mark.parametrize(
    "lane,arm,expected",
    [
        ("ax", "default", 75),
        ("ax", "long_deadline", 750),
        ("docker", "default", 30),
        ("docker", "long_deadline", 300),
    ],
)
def test_scaled_task_contract(lane, arm, expected):
    item = r.make_item({"id": "a"}, arm, "image", deadline_scale=2.5, lane=lane)
    spec = json.loads(item.inputs["SPEC_URL"])
    assert spec["hook_flags"] == ["--deadline", str(expected)]
    assert spec["deadline_seconds"] == expected and spec["deadline_scale"] == 2.5
    entry = importlib.import_module("benchmarks.unseen.replay_entry")
    entry.validate_spec({**spec, "repository_url": "https://public.example/test", "base": "a" * 40, "head": "b" * 40})


def test_native_sample_and_agreement():
    native = [timing_record(str(i)) for i in range(100)]
    cluster = [timing_record(str(i), lane="ax") for i in range(100)]
    assert r.sample_changes(native, 20) == r.sample_changes(list(reversed(native)), 20)
    report = s.agreement(cluster, native)
    assert report["rate"] == 1 and report["baseline_eligible"]
    assert s.agreement(cluster[:20], native[:20])["flagged"]  # Perfect small sample is insufficient.
    cluster[0]["push"]["quick_tier"][0]["findings"] = [{"cwe": "CWE-89"}]
    report = s.agreement(cluster, native)
    assert report["disagreements"] == ["0"] and report["rate"] == 0.99
    assert report["flagged"]
    cluster[0]["instrument"] = {}
    report = s.agreement(cluster, native)
    assert report["unknown"] == ["0"] and report["flagged"]
    assert s.agreement([], [])["flagged"]


def test_docker_error_capture_and_preflight(tmp_path, monkeypatch):
    calls = []
    monkeypatch.setenv("TEST_SECRET", "hidden-value")
    lane = object.__new__(b.DockerLane)
    lane.args = SimpleNamespace(docker_image_bytes=100)
    lane.workload = SimpleNamespace(image="pinned-image")
    lane._prepared = False

    def runner(command, **kwargs):
        calls.append(command)
        if command[:2] == ["docker", "info"]:
            return subprocess.CompletedProcess(command, 0, "/docker-data\n", "")
        if command[0] == "df":
            return subprocess.CompletedProcess(command, 0, "Avail\n2000000000\n", "")
        return subprocess.CompletedProcess(command, 125, "", "pull denied hidden-value https://host.example/?signature=private\nlast error")

    lane.runner = runner
    record = lane(None, tmp_path, 0)
    assert record["status"] == "instrument_failure"
    assert record["docker_errors"][0]["exit_code"] == 125
    assert record["docker_errors"][0]["stderr"].endswith("last error")
    assert "hidden-value" not in json.dumps(record) and "private" not in json.dumps(record)
    assert [cmd[0] for cmd in calls] == ["docker", "df", "docker"]
    assert calls[-1] == ["docker", "pull", "pinned-image"]
    assert (tmp_path / "docker-error.json").exists()
    lane.args.docker_image_bytes = 2000000000
    calls.clear()
    record = lane(None, tmp_path, 1)
    assert "refusing Docker pull" in record["docker_errors"][0]["stderr"]
    assert len(calls) == 2


def test_docker_launch_error_survives_transport(tmp_path, monkeypatch):
    monkeypatch.setenv("S3_ENDPOINT", "https://store.example")

    class Store:
        def _put(self, *args):
            pass

        def _delete(self, *args):
            pass

        def presign_get(self, *args):
            return "https://store.example/secret"

        presign_put = presign_get

    image = "registry/image@sha256:" + "a" * 64
    lane = b.DockerLane(
        SimpleNamespace(atespace="default"),
        b.Workload(image, frozenset({"SPEC_URL", "RESULT_URL"}), 60, None),
        store=Store(),
        runner=lambda command, **kw: subprocess.CompletedProcess(command, 0 if command[1] == "rm" else 125, "", "runtime missing"),
    )
    lane._prepared = True
    item = r.make_item({"id": "x"}, "default", image)
    record = lane(item, tmp_path, 0)
    assert record["docker_errors"] == [{"step": "run", "exit_code": 125, "stderr": "runtime missing"}]
    assert record["status"] == "instrument_failure"


def test_placement_scales_cluster_but_not_vm_fallback(tmp_path, monkeypatch):
    observed = []

    class Transport:
        def __init__(self, args, workload):
            assert workload.deadline >= 750 + 600

        def __call__(self, item, output, attempt):
            spec = json.loads(item.inputs["SPEC_URL"])
            observed.append(spec)
            return {"status": "quick_only"}

    def dispatch(items, workload, out, executors, **kwargs):
        for lane in ("ax", "docker"):
            record = executors[lane](items[0], out, 0)
            assert record["lane"] == lane
            assert record["deadline_seconds"] == (750 if lane == "ax" else 300)
        return False

    monkeypatch.setattr(r, "AXLane", Transport)
    monkeypatch.setattr(r, "DockerLane", Transport)
    monkeypatch.setattr(r, "dispatch", dispatch)
    args = SimpleNamespace(arm="long_deadline", image="test-image")
    assert not r.run_replays([{"id": "opaque"}], args, tmp_path, "mixed", deadline_scale=2.5)
    assert observed[0]["id"] == observed[1]["id"]
    assert observed[0]["hook_flags"] == ["--deadline", "750"]
    assert observed[1]["hook_flags"] == ["--deadline", "300"]


def test_calibration_cli_writes_anonymous_record(tmp_path, monkeypatch):
    manifest, record = tmp_path / "manifest.json", tmp_path / "record.json"
    changes = [{"id": str(i), "repo": "do-not-publish"} for i in range(25)]
    monkeypatch.setattr(c, "load_slice", lambda *args: changes)
    calls = []

    def replay(selected, args, out, lane, **kwargs):
        calls.append((selected, lane, kwargs))
        return False

    monkeypatch.setattr(c, "run_replays", replay)
    monkeypatch.setattr(
        c,
        "load_records",
        lambda path: [
            timing_record(str(i), factor=3 if path.name == "cluster" else 1, lane="ax" if path.name == "cluster" else "docker")
            for i in range(20)
        ],
    )
    monkeypatch.setattr(
        sys,
        "argv",
        [
            "calibrate",
            "--manifest",
            str(manifest),
            "--image",
            "digest",
            "--docker-image-bytes",
            "100",
            "--out",
            str(tmp_path / "raw"),
            "--record",
            str(record),
        ],
    )
    assert c.main() == 0
    assert len(calls[0][0]) == 20 and calls[0][0] == calls[1][0]
    assert [call[1] for call in calls] == ["docker", "ax"]
    assert all(call[2]["deadline"] == 600 for call in calls)
    payload = record.read_text()
    assert "do-not-publish" not in payload and "never-publish" not in payload
    assert json.loads(payload)["quick_total"]["factor"] == 3