mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Fable 5.1
f63703a604
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 <[email protected]>
142 lines
4.7 KiB
Python
142 lines
4.7 KiB
Python
"""Input provenance fails closed without dropping a declared evaluation case."""
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
from benchmarks.push.push_inputs import validate_manifest
|
|
|
|
|
|
def git(root, *args):
|
|
return subprocess.check_output(["git", "-C", str(root), *args]).decode().strip()
|
|
|
|
|
|
@pytest.fixture
|
|
def manifest(tmp_path):
|
|
source = tmp_path / "repo"
|
|
source.mkdir()
|
|
git(source, "init", "-q")
|
|
for name, value in [("LICENSE", "test license\n"), ("api.js", "eval(req.body.value);\n")]:
|
|
(source / name).write_text(value)
|
|
git(source, "add", ".")
|
|
git(source, "-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "input")
|
|
sha = git(source, "rev-parse", "HEAD")
|
|
cache = tmp_path / "cache" / "sample" / sha[:12]
|
|
cache.parent.mkdir(parents=True)
|
|
source.rename(cache)
|
|
recipe = tmp_path / "sample.toml"
|
|
recipe.write_text(
|
|
f'name="sample"\nurl="https://example.org/sample.git"\ncommit="{sha}"\nlanguage="javascript"\nlicense="MIT"\nlicense_file="LICENSE"\nmeasures=["envelope"]\n'
|
|
)
|
|
files = [
|
|
{"path": p.name, "sha256": hashlib.sha256(p.read_bytes()).hexdigest(), "size": p.stat().st_size}
|
|
for p in [cache / "LICENSE", cache / "api.js"]
|
|
]
|
|
data = {
|
|
"schema_version": 1,
|
|
"snapshots": [{"id": "vuln", "recipe": "sample.toml", "files": files}],
|
|
"cases": [
|
|
{
|
|
"id": "case",
|
|
"base": "vuln",
|
|
"tip": "vuln",
|
|
"label": "vulnerable",
|
|
"role": "development",
|
|
"capability": "javascript/express/injection",
|
|
"provenance": "upstream teaching example",
|
|
"prerequisites": [],
|
|
}
|
|
],
|
|
}
|
|
path = tmp_path / "inputs.json"
|
|
|
|
def run():
|
|
path.write_text(json.dumps(data))
|
|
return validate_manifest(path, cache_root=tmp_path / "cache")
|
|
|
|
return data, cache, run
|
|
|
|
|
|
def test_immutable_bytes_ignore_dirty_worktree(manifest):
|
|
_, root, run = manifest
|
|
(root / "api.js").write_text("different worktree\n")
|
|
result = run()
|
|
assert result["cases"][0]["status"] == "ready"
|
|
assert result["snapshots"][0]["bytes_read"] > 0
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["hash", "empty", "escape", "missing", "ref"])
|
|
def test_invalid_input_keeps_case_in_population(manifest, mutation):
|
|
data, _, run = manifest
|
|
snap = data["snapshots"][0]
|
|
file = snap["files"][1]
|
|
if mutation == "hash":
|
|
file["sha256"] = "0" * 64
|
|
elif mutation == "empty":
|
|
file["size"] = 0
|
|
elif mutation == "escape":
|
|
file["path"] = "../api.js"
|
|
elif mutation == "missing":
|
|
file["path"] = "missing.js"
|
|
else:
|
|
snap["commit"] = "main"
|
|
result = run()
|
|
assert result["population"] == 1
|
|
assert result["cases"][0]["status"] == "invalid_input"
|
|
|
|
|
|
def test_unavailable_source_is_explicit(manifest):
|
|
data, _, run = manifest
|
|
data["snapshots"][0]["commit"] = "a" * 40
|
|
result = run()
|
|
assert result["cases"][0]["status"] == "missing_prerequisite"
|
|
assert "fetch" in str(result)
|
|
|
|
|
|
def test_missing_holdout_labels_never_qualify_capability(manifest):
|
|
data, _, run = manifest
|
|
data["cases"][0].update(role="reserved_holdout", label="unreviewed", prerequisites=["review independent positive/fixed/benign pair"])
|
|
result = run()
|
|
assert result["cases"][0]["status"] == "missing_prerequisite"
|
|
assert result["cases"][0]["admission"] == "experimental"
|
|
|
|
|
|
def test_invalid_label_rejected(manifest):
|
|
data, _, run = manifest
|
|
data["cases"][0]["label"] = "clean"
|
|
assert run()["cases"][0]["status"] == "invalid_input"
|
|
|
|
|
|
def test_authored_edit_requires_exact_original_and_provenance(manifest):
|
|
data, _, run = manifest
|
|
f = data["snapshots"][0]["files"][1]
|
|
f["edit"] = {"before": "not in source", "after": "safe", "provenance": "authored repair"}
|
|
assert run()["cases"][0]["status"] == "invalid_input"
|
|
|
|
|
|
@pytest.mark.parametrize("field", ["snapshots", "cases"])
|
|
def test_empty_population_rejected(manifest, field):
|
|
from benchmarks.push.push_inputs import InputError
|
|
|
|
data, _, run = manifest
|
|
data[field] = []
|
|
with pytest.raises(InputError, match="nonempty"):
|
|
run()
|
|
|
|
|
|
def test_missing_snapshot_identity_rejected(manifest):
|
|
from benchmarks.push.push_inputs import InputError
|
|
|
|
data, _, run = manifest
|
|
del data["snapshots"][0]["id"]
|
|
with pytest.raises(InputError, match="identity"):
|
|
run()
|
|
|
|
|
|
def test_missing_language_holdouts_retained(manifest):
|
|
data, _, run = manifest
|
|
data["capability_prerequisites"] = {"php/wordpress/injection": ["untouched PHP pair missing"]}
|
|
assert run()["capability_prerequisites"] == data["capability_prerequisites"]
|