Open_Ultra_SAST/tests/test_push_inputs.py

"""Input provenance fails closed without dropping a declared evaluation case."""

import hashlib
import json
import subprocess

import pytest

from benchmarks.push.push_inputs import validate_manifest


def git(root, *args):
    return subprocess.check_output(["git", "-C", str(root), *args]).decode().strip()


@pytest.fixture
def manifest(tmp_path):
    source = tmp_path / "repo"
    source.mkdir()
    git(source, "init", "-q")
    for name, value in [("LICENSE", "test license\n"), ("api.js", "eval(req.body.value);\n")]:
        (source / name).write_text(value)
    git(source, "add", ".")
    git(source, "-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "input")
    sha = git(source, "rev-parse", "HEAD")
    cache = tmp_path / "cache" / "sample" / sha[:12]
    cache.parent.mkdir(parents=True)
    source.rename(cache)
    recipe = tmp_path / "sample.toml"
    recipe.write_text(
        f'name="sample"\nurl="https://example.org/sample.git"\ncommit="{sha}"\nlanguage="javascript"\nlicense="MIT"\nlicense_file="LICENSE"\nmeasures=["envelope"]\n'
    )
    files = [
        {"path": p.name, "sha256": hashlib.sha256(p.read_bytes()).hexdigest(), "size": p.stat().st_size}
        for p in [cache / "LICENSE", cache / "api.js"]
    ]
    data = {
        "schema_version": 1,
        "snapshots": [{"id": "vuln", "recipe": "sample.toml", "files": files}],
        "cases": [
            {
                "id": "case",
                "base": "vuln",
                "tip": "vuln",
                "label": "vulnerable",
                "role": "development",
                "capability": "javascript/express/injection",
                "provenance": "upstream teaching example",
                "prerequisites": [],
            }
        ],
    }
    path = tmp_path / "inputs.json"

    def run():
        path.write_text(json.dumps(data))
        return validate_manifest(path, cache_root=tmp_path / "cache")

    return data, cache, run


def test_immutable_bytes_ignore_dirty_worktree(manifest):
    _, root, run = manifest
    (root / "api.js").write_text("different worktree\n")
    result = run()
    assert result["cases"][0]["status"] == "ready"
    assert result["snapshots"][0]["bytes_read"] > 0


@pytest.mark.parametrize("mutation", ["hash", "empty", "escape", "missing", "ref"])
def test_invalid_input_keeps_case_in_population(manifest, mutation):
    data, _, run = manifest
    snap = data["snapshots"][0]
    file = snap["files"][1]
    if mutation == "hash":
        file["sha256"] = "0" * 64
    elif mutation == "empty":
        file["size"] = 0
    elif mutation == "escape":
        file["path"] = "../api.js"
    elif mutation == "missing":
        file["path"] = "missing.js"
    else:
        snap["commit"] = "main"
    result = run()
    assert result["population"] == 1
    assert result["cases"][0]["status"] == "invalid_input"


def test_unavailable_source_is_explicit(manifest):
    data, _, run = manifest
    data["snapshots"][0]["commit"] = "a" * 40
    result = run()
    assert result["cases"][0]["status"] == "missing_prerequisite"
    assert "fetch" in str(result)


def test_missing_holdout_labels_never_qualify_capability(manifest):
    data, _, run = manifest
    data["cases"][0].update(role="reserved_holdout", label="unreviewed", prerequisites=["review independent positive/fixed/benign pair"])
    result = run()
    assert result["cases"][0]["status"] == "missing_prerequisite"
    assert result["cases"][0]["admission"] == "experimental"


def test_invalid_label_rejected(manifest):
    data, _, run = manifest
    data["cases"][0]["label"] = "clean"
    assert run()["cases"][0]["status"] == "invalid_input"


def test_authored_edit_requires_exact_original_and_provenance(manifest):
    data, _, run = manifest
    f = data["snapshots"][0]["files"][1]
    f["edit"] = {"before": "not in source", "after": "safe", "provenance": "authored repair"}
    assert run()["cases"][0]["status"] == "invalid_input"


@pytest.mark.parametrize("field", ["snapshots", "cases"])
def test_empty_population_rejected(manifest, field):
    from benchmarks.push.push_inputs import InputError

    data, _, run = manifest
    data[field] = []
    with pytest.raises(InputError, match="nonempty"):
        run()


def test_missing_snapshot_identity_rejected(manifest):
    from benchmarks.push.push_inputs import InputError

    data, _, run = manifest
    del data["snapshots"][0]["id"]
    with pytest.raises(InputError, match="identity"):
        run()


def test_missing_language_holdouts_retained(manifest):
    data, _, run = manifest
    data["capability_prerequisites"] = {"php/wordpress/injection": ["untouched PHP pair missing"]}
    assert run()["capability_prerequisites"] == data["capability_prerequisites"]