mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 09:48:18 +00:00
353 lines
15 KiB
Python
353 lines
15 KiB
Python
"""Task 13.4 admission measurement unit tests (Reqs 12.1-12.9).
|
|
|
|
The admission runner consumes a reviewed upstream manifest for the untouched
|
|
pilot population, proves both sides of every vulnerable/fixed twin and every
|
|
reviewed ordinary change, maps per-site prover evidence onto the three-way
|
|
judgment vocabulary and returns the paired admission report.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import tomllib
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from benchmarks.unseen.replay import canonical # noqa: E402
|
|
from benchmarks.valid.admission_run import ( # noqa: E402
|
|
admission_ordinary,
|
|
admission_pair_cases,
|
|
assemble_admission_report,
|
|
classify_fixed_site,
|
|
coverage_share,
|
|
fixed_side_change,
|
|
load_admission_population,
|
|
locate_function,
|
|
site_judgment,
|
|
)
|
|
from benchmarks.valid.admission_scoring import AdmissionScoringError # noqa: E402
|
|
from benchmarks.valid.prover_admission import AdmissionManifest, AdmissionManifestError # noqa: E402
|
|
|
|
SITE = ("src/app.py", 30, 40)
|
|
|
|
|
|
def _trace(path: str, line: int, judgment: str, *, complete: bool = True) -> dict[str, object]:
|
|
return {
|
|
"fact_id": f"obligation:protected_read:{path}:{line}:identity_constraint",
|
|
"abstract_state": {
|
|
"path_judgment": {
|
|
"judgment": judgment,
|
|
"completeness": {"is_complete": complete, "reasons": [] if complete else ["deadline"]},
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _finding(path: str, line: int, family: str = "access_control") -> dict[str, object]:
|
|
return {"path": path, "line": line, "family": family, "id": f"obligation:{path}:{line}"}
|
|
|
|
|
|
def _record(record_id: str, **over: object) -> dict[str, Any]:
|
|
record = {
|
|
"id": record_id,
|
|
"repo": "example/project",
|
|
"kind": "vulnerability",
|
|
"path": "src/app.py",
|
|
"function_lines": [30, 40],
|
|
"findings": [],
|
|
"traces": [],
|
|
"degradations": [],
|
|
"stage_funnel": {"judgment": {"unguarded": 0, "discharged": 0, "indeterminate": 0}},
|
|
"instrument": {"hook_bytes_read": 10, "changed_bytes": {"src/app.py": 10}, "file_bytes": {"src/app.py": 10}},
|
|
"status": "prover_covered",
|
|
"seconds": 1.0,
|
|
}
|
|
record.update(over)
|
|
return record
|
|
|
|
|
|
def test_site_judgment_unguarded_when_finding_in_range() -> None:
|
|
record = _record("v1", findings=[_finding("src/app.py", 35)])
|
|
assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "unguarded"
|
|
|
|
|
|
def test_site_judgment_ignores_finding_outside_site() -> None:
|
|
record = _record("v1", findings=[_finding("src/other.py", 35), _finding("src/app.py", 200)])
|
|
assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "discharged"
|
|
|
|
|
|
def test_site_judgment_indeterminate_on_incomplete_trace() -> None:
|
|
record = _record("v1", traces=[_trace("src/app.py", 33, "indeterminate", complete=False)])
|
|
assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "indeterminate"
|
|
|
|
|
|
def test_site_judgment_discharged_on_completed_trace() -> None:
|
|
record = _record("v1", traces=[_trace("src/app.py", 33, "discharged", complete=True)])
|
|
assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "discharged"
|
|
|
|
|
|
def test_site_judgment_quiet_without_traces_is_discharged() -> None:
|
|
assert site_judgment(_record("v1"), path=SITE[0], start=SITE[1], end=SITE[2]) == "discharged"
|
|
|
|
|
|
def test_site_judgment_indeterminate_on_degraded_site_file() -> None:
|
|
record = _record("v1", degradations=[{"path": "src/app.py", "reason": "parse_failed"}])
|
|
assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "indeterminate"
|
|
|
|
|
|
def test_locate_function_by_name() -> None:
|
|
functions = [("helper", 1, 10, "function"), ("target", 30, 40, "function")]
|
|
assert locate_function(functions, name="target") == (30, 40)
|
|
assert locate_function(functions, name="absent") is None
|
|
|
|
|
|
def test_fixed_side_change_swaps_revisions() -> None:
|
|
change = {
|
|
"id": "case-1",
|
|
"base": "fixcommit",
|
|
"head": "vulnparent",
|
|
"files": [{"path": "src/app.py", "head_lines": [[30, 40]]}],
|
|
"kind": "vulnerability",
|
|
}
|
|
fixed = fixed_side_change(change, fixed_lines=(31, 42), fixed_operation="located")
|
|
assert fixed["head"] == "fixcommit"
|
|
assert fixed["base"] == "vulnparent"
|
|
assert fixed["files"] == [{"path": "src/app.py", "head_lines": [[31, 42]]}]
|
|
assert fixed["kind"] == "vulnerability"
|
|
assert fixed["fixed_lines"] == [31, 42]
|
|
|
|
|
|
def test_classify_fixed_site_missed_vs_removed() -> None:
|
|
# const-arrow / getter / moved-import: name present, locator failed.
|
|
assert classify_fixed_site(None, name_in_text=True) == "missed"
|
|
assert classify_fixed_site(None, name_in_text=False) == "removed"
|
|
assert classify_fixed_site((31, 42), name_in_text=True) == "located"
|
|
|
|
|
|
def test_pair_case_correct_uses_site_judgments_and_sums_seconds() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)], seconds=2.0, family="access_control")
|
|
fixed = _record(
|
|
"case-1",
|
|
kind="vulnerability",
|
|
fixed_lines=(31, 42),
|
|
traces=[_trace("src/app.py", 33, "discharged")],
|
|
seconds=3.0,
|
|
)
|
|
cases = admission_pair_cases((vulnerable,), (fixed,))
|
|
assert len(cases) == 1
|
|
assert cases[0].case_id == "case-1"
|
|
assert cases[0].vulnerable_judgment == "unguarded"
|
|
assert cases[0].fixed_judgment == "discharged"
|
|
assert cases[0].location_verified is True
|
|
assert cases[0].seconds == 5.0
|
|
|
|
|
|
def test_pair_case_wrong_family_is_not_location_verified() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35, family="output_encoding")])
|
|
fixed = _record("case-1", fixed_lines=(31, 42))
|
|
cases = admission_pair_cases((vulnerable,), (fixed,))
|
|
assert cases[0].vulnerable_judgment == "unguarded"
|
|
assert cases[0].location_verified is False
|
|
|
|
|
|
def test_removed_fixed_operation_is_quiet() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)])
|
|
fixed = _record("case-1", fixed_operation="removed")
|
|
cases = admission_pair_cases((vulnerable,), (fixed,))
|
|
assert cases[0].fixed_judgment == "discharged"
|
|
|
|
|
|
def test_locator_miss_fixed_operation_is_never_clean() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)])
|
|
fixed = _record("case-1", fixed_operation="missed")
|
|
cases = admission_pair_cases((vulnerable,), (fixed,))
|
|
assert cases[0].fixed_judgment == "indeterminate"
|
|
|
|
|
|
def test_unreadable_fixed_operation_is_indeterminate() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)])
|
|
fixed = _record("case-1")
|
|
cases = admission_pair_cases((vulnerable,), (fixed,))
|
|
assert cases[0].fixed_judgment == "indeterminate"
|
|
|
|
|
|
def test_ordinary_change_flagged_on_any_finding() -> None:
|
|
record = _record("ord-1", kind="ordinary", findings=[_finding("src/any.py", 5)])
|
|
changes = admission_ordinary((record,))
|
|
assert changes[0].judgment == "unguarded"
|
|
assert changes[0].mislabeled_defect is False
|
|
|
|
|
|
def test_ordinary_change_quiet_and_indeterminate() -> None:
|
|
quiet = admission_ordinary((_record("ord-1", kind="ordinary"),))[0]
|
|
assert quiet.judgment == "discharged"
|
|
funnel = {"judgment": {"unguarded": 0, "discharged": 0, "indeterminate": 2}}
|
|
undecided = admission_ordinary((_record("ord-2", kind="ordinary", stage_funnel=funnel),))[0]
|
|
assert undecided.judgment == "indeterminate"
|
|
|
|
|
|
def test_coverage_share_over_repositories() -> None:
|
|
covered = _record("a", repo="repo-a", stage_funnel={"entry_recognition": {"frameworks_recognised": {"jenkins": 1}}})
|
|
bare = _record("b", repo="repo-b", stage_funnel={"entry_recognition": {"frameworks_recognised": {}}})
|
|
other = _record("a", repo="repo-a", stage_funnel={"entry_recognition": {"frameworks_recognised": {"jenkins": 2}}})
|
|
assert coverage_share((covered, bare, other)) == 0.5
|
|
|
|
|
|
def _manifest_payload(**over: object) -> dict[str, object]:
|
|
payload = {
|
|
"produced_by": "valid-detection-benchmark",
|
|
"reviewed": True,
|
|
"label_isolated": True,
|
|
"spent": False,
|
|
"population_role": "admission",
|
|
"analyzer_revision": "d063167",
|
|
"fact_set_digest": "3eb5bf699434",
|
|
"acquisition_record_digest": "none",
|
|
"configuration_digest": "be6d0f688598",
|
|
"population_digest": "0" * 64,
|
|
"labels": [],
|
|
}
|
|
payload.update(over)
|
|
return payload
|
|
|
|
|
|
POOL_TOML = (
|
|
'status = "frozen"\n'
|
|
'pool = "valid"\n'
|
|
"seed = 0\n"
|
|
"\n"
|
|
"[[repository]]\n"
|
|
'id = "example/project"\n'
|
|
'repository = "example/project"\n'
|
|
'url = "https://github.com/example/project.git"\n'
|
|
"slice = 1\n"
|
|
"changes = [\n"
|
|
' {base = "fixcommit", head = "vulnparent", case_id = "example-project-vuln-1", '
|
|
'id = "example-project-vuln-1", repo = "example/project", kind = "vulnerability", '
|
|
'path = "src/app.py", function = "handler", function_lines = [30, 40], '
|
|
'files = [{path = "src/app.py", head_lines = [[30, 40]]}], family = "access_control"},\n'
|
|
' {base = "parent1", head = "ordinary1", case_id = "example-project-ordinary-1", '
|
|
'id = "example-project-ordinary-1", repo = "example/project", kind = "ordinary", '
|
|
'files = [{path = "src/lib.py", head_lines = [[1, 5]]}]},\n'
|
|
"]\n"
|
|
)
|
|
|
|
|
|
def _population_files(root: Path, *, freeze_digest: str | None = None, guard_result: int = 0) -> tuple[Path, Path, str]:
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
pool_path = root / "pool.toml"
|
|
pool_path.write_text(POOL_TOML, encoding="utf-8")
|
|
pool = tomllib.loads(pool_path.read_text(encoding="utf-8"))
|
|
digest = freeze_digest if freeze_digest is not None else hashlib.sha256(canonical(pool)).hexdigest()
|
|
freeze_path = root / "freeze.json"
|
|
freeze_path.write_text(json.dumps({"freeze_digest": digest, "guard_result": guard_result}), encoding="utf-8")
|
|
return pool_path, freeze_path, digest
|
|
|
|
|
|
def test_load_population_verifies_freeze_digest_and_guard() -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
pool_path, freeze_path, digest = _population_files(root)
|
|
payload = _manifest_payload(population_digest=digest)
|
|
vulnerable, ordinary = load_admission_population(payload, pool_path, freeze_path)
|
|
assert [change["kind"] for change in (*vulnerable, *ordinary)] == ["vulnerability", "ordinary"]
|
|
assert all(len(change["id"]) == 64 for change in (*vulnerable, *ordinary))
|
|
tampered = _manifest_payload(population_digest="f" * 64)
|
|
try:
|
|
load_admission_population(tampered, pool_path, freeze_path)
|
|
raise AssertionError("population digest mismatch must be refused")
|
|
except ValueError as error:
|
|
assert "population" in str(error)
|
|
guarded = _population_files(root / "g", guard_result=1)
|
|
payload_guarded = _manifest_payload(population_digest=guarded[2])
|
|
try:
|
|
load_admission_population(payload_guarded, guarded[0], guarded[1])
|
|
raise AssertionError("held-out guard failure must be refused")
|
|
except ValueError as error:
|
|
assert "guard" in str(error)
|
|
|
|
|
|
def test_load_population_validates_manifest_before_labels() -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
pool_path, freeze_path, _ = _population_files(root)
|
|
spent = _manifest_payload(spent=True)
|
|
try:
|
|
load_admission_population(spent, pool_path, freeze_path)
|
|
raise AssertionError("spent population must be refused")
|
|
except AdmissionManifestError:
|
|
pass
|
|
|
|
|
|
def _admission_manifest(digest: str = "0" * 64) -> AdmissionManifest:
|
|
from benchmarks.valid.prover_admission import validate_admission_manifest
|
|
|
|
return validate_admission_manifest(_manifest_payload(population_digest=digest))
|
|
|
|
|
|
def test_assemble_report_returns_go_without_failed_gates() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)], family="access_control")
|
|
fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")])
|
|
ordinary = _record("ord-1", kind="ordinary")
|
|
manifest = _admission_manifest()
|
|
report = assemble_admission_report(
|
|
manifest,
|
|
(vulnerable,),
|
|
(fixed,),
|
|
(ordinary,),
|
|
criteria={"min_fix_recall": 1.0, "max_flagged_ordinary_per_100": 0.0},
|
|
)
|
|
assert report["decision"] == "GO"
|
|
assert report["fix_recall"]["count"] == 1
|
|
assert report["pairs"] == {"n": 1, "correct": 1}
|
|
assert report["flagged_ordinary_per_100"]["count"] == 0
|
|
|
|
|
|
def test_assemble_report_no_go_on_ordinary_overrun() -> None:
|
|
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)])
|
|
fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")])
|
|
slow = _record("ord-1", kind="ordinary", seconds=300.0)
|
|
report = assemble_admission_report(_admission_manifest(), (vulnerable,), (fixed,), (slow,), criteria=None, budget_seconds=120.0)
|
|
assert report["decision"] == "NO-GO"
|
|
assert any(gate.startswith("budget:ord-1:") for gate in report["failed_gates"])
|
|
|
|
|
|
def test_assemble_report_no_go_on_budget_and_criteria() -> None:
|
|
vulnerable = _record("case-1", seconds=200.0)
|
|
fixed = _record("case-1", fixed_lines=(31, 42), seconds=1.0)
|
|
ordinary = _record("ord-1", kind="ordinary", findings=[_finding("src/x.py", 3)])
|
|
manifest = _admission_manifest()
|
|
report = assemble_admission_report(
|
|
manifest,
|
|
(vulnerable,),
|
|
(fixed,),
|
|
(ordinary,),
|
|
criteria={"min_fix_recall": 0.9, "max_flagged_ordinary_per_100": 0.0},
|
|
)
|
|
assert report["decision"] == "NO-GO"
|
|
assert any(gate.startswith("budget:case-1:") for gate in report["failed_gates"])
|
|
assert any("min_fix_recall" in gate for gate in report["failed_gates"])
|
|
assert any("max_flagged_ordinary_per_100" in gate for gate in report["failed_gates"])
|
|
|
|
|
|
def test_assemble_report_counts_instrument_failures() -> None:
|
|
vulnerable = _record("case-1", status="instrument_failure")
|
|
fixed = _record("case-1", fixed_lines=(31, 42), status="instrument_failure")
|
|
manifest = _admission_manifest()
|
|
report = assemble_admission_report(manifest, (vulnerable,), (fixed,), (), criteria=None)
|
|
assert report["instrument_failures"] == 2
|
|
assert report["decision"] == "NO-GO"
|
|
|
|
|
|
def test_assemble_report_refuses_development_role() -> None:
|
|
manifest = _admission_manifest()
|
|
object.__setattr__(manifest, "population_role", "development")
|
|
try:
|
|
assemble_admission_report(manifest, (), (), (), criteria=None)
|
|
raise AssertionError("development population must be refused for admission")
|
|
except AdmissionScoringError:
|
|
pass
|