mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
unseen-eval groups 4-6: generic AX batch dispatcher (engine_trace on it), replay task and image stage, scorer with adoption gate
Implemented with Codex. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
48ae918dec
commit
c6e87a8517
@@ -0,0 +1,159 @@
|
||||
"""Offline shared dispatcher contracts."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.ax.batch import Item, Workload, dispatch, manifest
|
||||
|
||||
IMAGE = "registry/task@sha256:" + "a" * 64
|
||||
|
||||
|
||||
def item(identity="one"):
|
||||
return Item(identity, {"SPEC_URL": b"{}"}, "result.dat", ("python3", "/entry.py"), {}, identity)
|
||||
|
||||
|
||||
def workload():
|
||||
return Workload(IMAGE, frozenset({"SPEC_URL", "RESULT_URL"}), 900, lambda data, output: {})
|
||||
|
||||
|
||||
def test_manifest_contract():
|
||||
doc = manifest(
|
||||
"ousast-engine-replay-abc",
|
||||
workload(),
|
||||
item(),
|
||||
{"SPEC_URL": "https://files.example/spec.dat", "RESULT_URL": "https://files.example/result.dat"},
|
||||
)
|
||||
assert {v["name"] for v in doc["spec"]["env"]} == {"SPEC_URL", "RESULT_URL"}
|
||||
with pytest.raises(ValueError):
|
||||
manifest("bad", workload(), item(), {})
|
||||
bad = item()
|
||||
bad.extra_env["AWS_SECRET_ACCESS_KEY"] = "secret"
|
||||
with pytest.raises(ValueError):
|
||||
manifest(
|
||||
"ousast-engine-test",
|
||||
workload(),
|
||||
bad,
|
||||
{"SPEC_URL": "https://files.example/spec.dat", "RESULT_URL": "https://files.example/result.dat"},
|
||||
)
|
||||
|
||||
|
||||
def test_dispatch_retry_fallback_resume_stop(tmp_path):
|
||||
calls = []
|
||||
|
||||
def lane(it, output, attempt):
|
||||
calls.append(("ax", attempt))
|
||||
return {"status": "sandbox_failure"}
|
||||
|
||||
def vm(it, output, attempt):
|
||||
calls.append(("docker", attempt))
|
||||
return {"status": "ok"}
|
||||
|
||||
assert not dispatch([item()], workload(), tmp_path, {"ax": lane, "docker": vm}, lanes=("ax", "ax", "docker"), overflow=False)
|
||||
assert [c[0] for c in calls] == ["ax", "ax", "docker"]
|
||||
assert not dispatch([item()], workload(), tmp_path, {"ax": lane, "docker": vm})
|
||||
assert len(calls) == 3
|
||||
(tmp_path / "STOP").touch()
|
||||
dispatch([item("two")], workload(), tmp_path, {"ax": lane, "docker": vm})
|
||||
assert len(calls) == 3
|
||||
assert json.loads((tmp_path / "progress.json").read_text())["pins_in_flight"] == 0
|
||||
|
||||
|
||||
def test_cleanup_failure_halts(tmp_path):
|
||||
calls = []
|
||||
|
||||
def broken(it, output, attempt):
|
||||
calls.append(it.id)
|
||||
raise RuntimeError("cleanup failed")
|
||||
|
||||
with pytest.raises(RuntimeError, match="cleanup"):
|
||||
dispatch([item("one"), item("two")], workload(), tmp_path, {"ax": broken}, lanes=("ax",))
|
||||
assert calls == ["one"]
|
||||
|
||||
|
||||
def test_ax_lane_fake_cli_and_store(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
from benchmarks.ax.batch import AXLane
|
||||
|
||||
calls = []
|
||||
|
||||
class Store:
|
||||
objects = {}
|
||||
|
||||
def _put(self, key, data):
|
||||
self.objects[key] = data
|
||||
|
||||
def _get(self, key):
|
||||
return (b"completed", None)
|
||||
|
||||
def _delete(self, key):
|
||||
self.objects.pop(key, None)
|
||||
|
||||
def presign_get(self, key, expiry):
|
||||
assert key.endswith(".dat")
|
||||
return "https://files.example/" + key + "?signature=secret"
|
||||
|
||||
presign_put = presign_get
|
||||
|
||||
def run(command, **kwargs):
|
||||
calls.append((command, kwargs))
|
||||
if command[1] == "get":
|
||||
name = json.loads(calls[0][1]["input"])["metadata"]["name"]
|
||||
return subprocess.CompletedProcess(command, 0, json.dumps({"metadata": {"name": name}, "status": {"phase": "Running"}}), "")
|
||||
return subprocess.CompletedProcess(command, 0, "", "")
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", run)
|
||||
monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
|
||||
|
||||
def validate(data, output):
|
||||
assert data == b"completed"
|
||||
(output / "result.json").write_text('{"status":"ok"}')
|
||||
|
||||
work = Workload(IMAGE, frozenset({"SPEC_URL", "RESULT_URL"}), 10, validate)
|
||||
lane = AXLane(SimpleNamespace(ax_bin="fake-ax", kubeconfig=None, atespace="default"), work, Store())
|
||||
assert lane(item(), tmp_path, 0)["status"] == "ok"
|
||||
doc = json.loads(calls[0][1]["input"])
|
||||
assert doc["metadata"]["name"].startswith("ousast-engine-") and len(doc["metadata"]["name"]) <= 49
|
||||
assert {e["name"] for e in doc["spec"]["env"]} == {"SPEC_URL", "RESULT_URL"}
|
||||
assert calls[-1][0][1:3] == ["delete", "task"]
|
||||
assert not lane.store.objects
|
||||
|
||||
|
||||
def test_bounded_lanes_oom_and_instrument_retry(tmp_path):
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
active, peak, calls = Counter(), Counter(), []
|
||||
lock = threading.Lock()
|
||||
|
||||
def execute(lane):
|
||||
def run(it, output, attempt):
|
||||
with lock:
|
||||
calls.append((it.id, lane, attempt))
|
||||
active[lane] += 1
|
||||
peak[lane] = max(peak[lane], active[lane])
|
||||
time.sleep(0.01)
|
||||
with lock:
|
||||
active[lane] -= 1
|
||||
return {"status": "oom" if lane == "ax" else "ok"}
|
||||
|
||||
return run
|
||||
|
||||
assert not dispatch([item(str(i)) for i in range(9)], workload(), tmp_path, {"ax": execute("ax"), "docker": execute("docker")})
|
||||
assert peak["ax"] == 2 and peak["docker"] == 1
|
||||
assert all(attempt == 0 for _, _, attempt in calls)
|
||||
attempts = []
|
||||
|
||||
def failed(it, output, attempt):
|
||||
attempts.append(attempt)
|
||||
return {"status": "instrument_failure"}
|
||||
|
||||
assert dispatch([item("fail")], workload(), tmp_path, {"ax": failed}, lanes=("ax",))
|
||||
assert attempts == [0, 1]
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Static image contract; no Docker invocation."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ENGINE_SHA = "06ec67d411cd7abbfcd0735b1c2a70e4f3eb65ddcfeef89975709c664f59ad42"
|
||||
|
||||
|
||||
def test_image_stages():
|
||||
text = (ROOT / "plane/Dockerfile.engine-task").read_text()
|
||||
engine, replay = text.split("FROM engine AS replay", 1)
|
||||
original = engine.replace("v0.3.1 AS engine", "v0.3.1").rstrip() + "\n"
|
||||
assert hashlib.sha256(original.encode()).hexdigest() == ENGINE_SHA
|
||||
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS engine" in engine
|
||||
assert "git" in replay and "replay_entry.py" in replay and "ousast pre-push --help" in replay
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Proof records fail closed using synthetic local results."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
p = importlib.import_module("benchmarks.unseen.proof")
|
||||
|
||||
|
||||
def test_replay_proof(tmp_path):
|
||||
roots = {}
|
||||
for lane in ("kind", "ax", "vm"):
|
||||
root = tmp_path / lane
|
||||
root.mkdir()
|
||||
roots[lane] = root
|
||||
(root / "inputs.json").write_text("[]")
|
||||
for i in range(5):
|
||||
(root / f"{i}.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"input_digest": str(i),
|
||||
"status": "engine_covered",
|
||||
"instrument": {"hook_bytes_read": 20, "jvm_wall_seconds": 8, "seconds": 30, "hook_exit": 0},
|
||||
"push": {"quick_tier": [{"findings": []}]},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert p.replay_proof(roots)["quick_tier_equal"]
|
||||
path = roots["vm"] / "0.json"
|
||||
record = json.loads(path.read_text())
|
||||
record["instrument"]["jvm_wall_seconds"] = 0
|
||||
path.write_text(json.dumps(record))
|
||||
with pytest.raises(ValueError, match="instrument"):
|
||||
p.replay_proof(roots)
|
||||
|
||||
|
||||
def test_engine_proof_counts_differences_without_names(tmp_path):
|
||||
current, baseline = tmp_path / "current", tmp_path / "baseline"
|
||||
current.mkdir()
|
||||
baseline.mkdir()
|
||||
for i in range(2):
|
||||
record = {
|
||||
"repo": "synthetic",
|
||||
"pin": str(i),
|
||||
"done": True,
|
||||
"seconds": 20,
|
||||
"units": [{"unit": str(i), "status": "path", "instrument": {"bytes": 20, "jvm": [{"seconds": 8, "success": True}]}}],
|
||||
}
|
||||
(current / f"{i}.json").write_text(json.dumps(record))
|
||||
(baseline / f"{i}.json").write_text(json.dumps(record))
|
||||
proof = p.engine_proof(current, baseline)
|
||||
assert proof["differences"] == 0 and "synthetic" not in json.dumps(proof)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Replay workload privacy, validation and input identity."""
|
||||
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
r = importlib.import_module("benchmarks.unseen.replay")
|
||||
|
||||
|
||||
def test_items_private_and_arm_identity():
|
||||
change = {
|
||||
"id": "repo-name-secret",
|
||||
"repo": "owner/private-name",
|
||||
"repository_url": "https://public.example/private-name",
|
||||
"base": "a" * 40,
|
||||
"head": "b" * 40,
|
||||
"files": [{"path": "app.py", "head_lines": [[1, 2]]}],
|
||||
}
|
||||
image = "registry/replay@sha256:" + "a" * 64
|
||||
a = r.make_item(change, "default", image)
|
||||
b = r.make_item(change, "long_deadline", image)
|
||||
assert a.input_digest != b.input_digest
|
||||
assert "private-name" not in a.id and "repo-name" not in a.id
|
||||
assert set(a.inputs) == {"SPEC_URL"} and not a.extra_env
|
||||
spec = json.loads(a.inputs["SPEC_URL"])
|
||||
assert spec["repository_url"] == change["repository_url"]
|
||||
assert json.loads(b.inputs["SPEC_URL"])["hook_flags"] == ["--deadline", "300"]
|
||||
|
||||
|
||||
def test_validator_requires_instrument(tmp_path):
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode="w") as tar:
|
||||
data = b"{}"
|
||||
info = tarfile.TarInfo("push.json")
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
with pytest.raises(ValueError, match="instrument"):
|
||||
r.validate_result(buffer.getvalue(), tmp_path)
|
||||
|
||||
|
||||
def test_frozen_toml_slice_and_scoring_join(tmp_path):
|
||||
import hashlib
|
||||
|
||||
from benchmarks.unseen.score import load_records
|
||||
|
||||
pool = tmp_path / "pool-p1.toml"
|
||||
pool.write_text("""pool="p1"
|
||||
[[repository]]
|
||||
id="opaque-a"
|
||||
slice=1
|
||||
url="https://public.example/repo-a"
|
||||
changes=[{kind="ordinary",base="aaa",head="bbb",files=[]}]
|
||||
[[repository]]
|
||||
id="opaque-b"
|
||||
slice=2
|
||||
url="https://public.example/repo-b"
|
||||
changes=[{kind="ordinary",base="ccc",head="ddd",files=[]}]
|
||||
""")
|
||||
import tomllib
|
||||
|
||||
freeze = hashlib.sha256(r.canonical(tomllib.loads(pool.read_text()))).hexdigest()
|
||||
(tmp_path / "freeze-p1.json").write_text(json.dumps({"freeze_digest": freeze}))
|
||||
changes = r.load_slice(pool, 2)
|
||||
assert len(changes) == 1 and changes[0]["repo"] == "opaque-b"
|
||||
(tmp_path / "inputs.json").write_text(json.dumps([{"item_id": "replay-opaque", "change": changes[0], "input_digest": "digest"}]))
|
||||
assert load_records(tmp_path)[0]["kind"] == "ordinary"
|
||||
assert "instrument" not in load_records(tmp_path)[0] # Missing result remains a measured failure.
|
||||
result = tmp_path / (hashlib.sha256(b"replay-opaque").hexdigest() + ".json")
|
||||
result.write_text(json.dumps({"input_digest": "digest", "instrument": {"head_verified": True}}))
|
||||
assert load_records(tmp_path)[0]["instrument"]["head_verified"]
|
||||
pool.write_text(pool.read_text() + "changed=true\n")
|
||||
with pytest.raises(ValueError, match="digest"):
|
||||
r.load_slice(pool, 2)
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Replay entry uses real local git and an injected transfer, never a network."""
|
||||
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
e = importlib.import_module("benchmarks.unseen.replay_entry")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_repo(tmp_path):
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
|
||||
def git(*args):
|
||||
return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
git("init", "-q")
|
||||
git("config", "user.email", "[email protected]")
|
||||
git("config", "user.name", "Fixture")
|
||||
(repo / "app.py").write_text("print(1)\n")
|
||||
git("add", "app.py")
|
||||
git("commit", "-qm", "base")
|
||||
base = git("rev-parse", "HEAD")
|
||||
(repo / "app.py").write_text("print(2)\n")
|
||||
git("commit", "-qam", "head")
|
||||
return repo, base, git("rev-parse", "HEAD")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [None, "missing", "fetch", "crash", "fast", "unread", "unsupported"])
|
||||
def test_entry(local_repo, tmp_path, monkeypatch, failure):
|
||||
repo, base, head = local_repo
|
||||
spec = {
|
||||
"repository_url": "https://public.example/project.git",
|
||||
"base": base,
|
||||
"head": head,
|
||||
"id": "opaque",
|
||||
"image": "digest",
|
||||
"hook_flags": [],
|
||||
}
|
||||
if failure == "missing":
|
||||
spec["head"] = "a" * 40
|
||||
received = {}
|
||||
|
||||
def transfer(request, destination=None, **kwargs):
|
||||
assert kwargs["readiness_budget"] == 60
|
||||
if destination:
|
||||
destination.write_text(json.dumps(spec))
|
||||
else:
|
||||
with tarfile.open(fileobj=io.BytesIO(request.data)) as archive:
|
||||
received.update({m.name: archive.extractfile(m).read() for m in archive.getmembers()})
|
||||
|
||||
monkeypatch.setenv("SPEC_URL", "https://files.example/spec.dat?secret")
|
||||
monkeypatch.setenv("RESULT_URL", "https://files.example/result.dat?secret")
|
||||
monkeypatch.setattr(e, "transfer", transfer)
|
||||
real_run = subprocess.run
|
||||
|
||||
def run(command, **kwargs):
|
||||
if command[0] == "ousast":
|
||||
assert command[command.index("--base") + 1] == base
|
||||
assert kwargs["env"]["OPENULTRASAST_CPG_HEAP_MB"] == "1700"
|
||||
if failure == "crash":
|
||||
return subprocess.CompletedProcess(command, 2, "", "crash https://secret")
|
||||
target = Path(command[command.index("--artifact") + 1])
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"quick_tier": [{"files": ["app.py"], "bytes_read": 0 if failure == "unread" else 9, "findings": []}],
|
||||
"scans": [
|
||||
{
|
||||
"side": "head",
|
||||
"scan": {"build_seconds": 0.1 if failure == "fast" else 8, "query_seconds": 2, "question_outcomes": []},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
if failure == "unsupported":
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"quick_tier": [{"files": [], "bytes_read": 0, "findings": []}],
|
||||
"admission": {"coverage_reasons": ["language_not_covered:rust:1:1"]},
|
||||
"scans": [
|
||||
{
|
||||
"side": "head",
|
||||
"scan": {
|
||||
"build_seconds": 0.01,
|
||||
"query_seconds": 0,
|
||||
"partitions": [
|
||||
{
|
||||
"language": "rust",
|
||||
"frontend": None,
|
||||
"paths": ["app.rs"],
|
||||
"source_bytes": 0,
|
||||
"status": "unsupported",
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
return subprocess.CompletedProcess(command, 0, "Skipped: deadline\n", "")
|
||||
command = [
|
||||
str(repo) if arg == spec["repository_url"] else "protocol.file.allow=always" if arg == "protocol.file.allow=never" else arg
|
||||
for arg in command
|
||||
]
|
||||
if failure == "fetch" and "fetch" in command:
|
||||
return subprocess.CompletedProcess(command, 1, "", "fetch secret")
|
||||
return real_run(command, **kwargs)
|
||||
|
||||
result = e.main(work_root=tmp_path / "work", runner=run)
|
||||
instrument = json.loads(received["instrument.json"])
|
||||
assert instrument["image"] == "digest"
|
||||
assert b"secret" not in received["instrument.json"]
|
||||
if failure == "unsupported":
|
||||
from benchmarks.unseen.score import coverage
|
||||
|
||||
assert result == 0 and not instrument["suspect_fast"]
|
||||
assert coverage({"instrument": instrument, "push": json.loads(received["push.json"])}) == "unanalysable"
|
||||
elif failure:
|
||||
assert result == 1 and instrument["status"] == "instrument_failure"
|
||||
else:
|
||||
assert result == 0 and instrument["changed_bytes"] == {"app.py": 9}
|
||||
assert instrument["head_verified"] and instrument["base_verified"]
|
||||
assert instrument["jvm_wall_seconds"] == 10
|
||||
assert b"Skipped:" in received["push.txt"]
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Synthetic push-level statistical controls; no corpus reads."""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
s = importlib.import_module("benchmarks.unseen.score")
|
||||
|
||||
|
||||
def record(kind="introducing", **updates):
|
||||
return {
|
||||
"id": "a",
|
||||
"repo": "opaque",
|
||||
"kind": kind,
|
||||
"family": "injection",
|
||||
"function": "handler",
|
||||
"files": [{"path": "app.py", "head_lines": [[20, 22]]}],
|
||||
"instrument": {
|
||||
"head_verified": True,
|
||||
"base_verified": True,
|
||||
"changed_bytes": {"app.py": 20},
|
||||
"hook_exit": 0,
|
||||
"hook_bytes_read": 20,
|
||||
},
|
||||
"push": {"quick_tier": [{"bytes_read": 20, "findings": []}]},
|
||||
**updates,
|
||||
}
|
||||
|
||||
|
||||
def test_coverage_and_failure_denominator():
|
||||
r = record()
|
||||
assert s.coverage(r) == "quick_only"
|
||||
r["push"]["scans"] = [{"side": "head", "scan": {"question_outcomes": [{"status": "completed", "identity": {"path": "app.py"}}]}}]
|
||||
assert s.coverage(r) == "engine_covered"
|
||||
r["instrument"]["suspect_fast"] = True
|
||||
assert s.coverage(r) == "instrument_failure"
|
||||
assert s.coverage(record(push={"skipped": ["language_not_covered:other:1:1"]})) == "unanalysable"
|
||||
assert s.coverage({}) == "instrument_failure"
|
||||
with pytest.raises(ValueError, match="5%"):
|
||||
s.score([record(), r])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"p,width,n", [(0.05, 0.03, 315), (0.1, 0.03, 483), (0.2, 0.03, 756), (0.1, 0.1, 62), (0.25, 0.1, 88), (0.4, 0.1, 97)]
|
||||
)
|
||||
def test_sample_sizes(p, width, n):
|
||||
assert s.required_n(p, width) == n
|
||||
|
||||
|
||||
def test_catch_and_all_stream_false_alarm():
|
||||
r = record()
|
||||
r["push"]["quick_tier"][0]["findings"] = [{"cwe": "CWE-89", "path": "app.py", "line": 20}]
|
||||
assert s.catches(r) == ["quick"]
|
||||
r["push"]["quick_tier"][0]["findings"][0]["line"] = 100
|
||||
assert not s.catches(r)
|
||||
r["kind"] = "ordinary"
|
||||
assert s.false_alarm(r)
|
||||
assert not s.false_alarm(r, alerts_only=True)
|
||||
|
||||
|
||||
def test_location_rules():
|
||||
assert s.location_match(record(family="access_control"), {"path": "app.py", "function": "handler", "line": 200})
|
||||
assert not s.location_match(record(family="config_secrets"), {"path": "app.py", "function": "handler", "line": 200})
|
||||
assert s.location_match(record(family="config_secrets"), {"path": "app.py", "line": 30})
|
||||
assert not s.location_match(record(), {"path": "else.py", "function": "handler", "line": 20})
|
||||
|
||||
|
||||
def test_bootstrap_icc_mcnemar():
|
||||
rows = [("a", 1), ("a", 1), ("b", 0), ("b", 0)]
|
||||
assert s.cluster_interval(rows, seed=7) == s.cluster_interval(rows, seed=7)
|
||||
assert s.icc(rows) == pytest.approx(1)
|
||||
assert s.mcnemar([True] * 6, [False] * 6) == pytest.approx(0.03125)
|
||||
assert s.mcnemar([True], [True]) == 1
|
||||
|
||||
|
||||
def test_budgets():
|
||||
rows = [{"kind": "ordinary", "score": x} for x in range(100)] + [{"kind": "introducing", "score": 99}]
|
||||
result = s.budgets(rows)
|
||||
assert result["5%"]["threshold"] == 95
|
||||
assert result["5%"]["label"] == "exploratory"
|
||||
assert s.budgets(rows, thresholds={"5%": 96})["5%"]["label"] == "fixed"
|
||||
binary = s.budgets([{"kind": "ordinary", "prediction": False}, {"kind": "introducing", "prediction": True}], binary=True)
|
||||
assert binary["budgets_met"] == ["1%", "5%", "10%"]
|
||||
|
||||
|
||||
def test_gate():
|
||||
a = [record(kind="ordinary", id=str(i), repo=str(i)) for i in range(10)] + [record(id="v")]
|
||||
b = [
|
||||
record(kind="ordinary", id=str(i), repo=str(i), push={"quick_tier": [{"findings": [{"cwe": "CWE-89", "path": "x", "line": 1}]}]})
|
||||
for i in range(10)
|
||||
] + [record(id="v")]
|
||||
assert s.gate(a, b, slice_id="p1/2")["gate"] == "regression"
|
||||
assert s.gate(a, b)["gate"] == "not_run"
|
||||
assert s.gate(a, b, slice_id="p1/2", informed=True)["gate"] == "not_run"
|
||||
assert s.gate(a, [{}], slice_id="p1/2")["gate"] == "not_run"
|
||||
|
||||
|
||||
def test_real_artifact_coverage_and_location_binding():
|
||||
r = record(path="app.py", function_lines=[1, 100], files=[{"path": "other.py", "head_lines": [[200, 210]]}])
|
||||
assert not s.location_match(r, {"path": "other.py", "line": 40})
|
||||
assert (
|
||||
s.coverage(
|
||||
record(
|
||||
instrument={"head_verified": True, "base_verified": True, "changed_bytes": {"x": 20}, "hook_exit": 0, "hook_bytes_read": 0},
|
||||
push={"admission": {"coverage_reasons": ["language_not_covered:unknown:1:1"]}},
|
||||
)
|
||||
)
|
||||
== "unanalysable"
|
||||
)
|
||||
r["push"]["quick_tier"][0]["findings"] = [{"cwe": None, "path": "other.py", "line": 200}]
|
||||
assert not s.catches(r)
|
||||
assert s.false_alarm(r)
|
||||
|
||||
|
||||
def test_committed_synthetic_fixture():
|
||||
import json
|
||||
|
||||
fixture = Path(__file__).resolve().parents[1] / "benchmarks/unseen/fixtures/scorer.json"
|
||||
payload = fixture.read_bytes()
|
||||
assert len(payload) > 0
|
||||
report = s.score(json.loads(payload))
|
||||
assert report["catch"]["n"] == 4 and report["catch"]["count"] == 2
|
||||
assert report["false_alarm"]["n"] == 16 and report["false_alarm"]["count"] == 1
|
||||
assert report["coverage"]["quick_only"] == 20
|
||||
Reference in New Issue
Block a user