mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
unseen: implement freeze p1 (group 3: bisect guard, freeze writer, protocol)
Group 3 was unimplemented, so there was no way to turn the draft into a frozen pool. Add: - bisect_guard.py: stages the cache draft and bisects it against the reserved-repository guard, run as a black box (pytest -k referenced, stdout/stderr discarded, exit code only; an exit other than 0/1 aborts as an instrument failure). Drops guard-tripping candidates and reports the dropped COUNT only -- never a name or URL. Guard runner is injectable for offline tests. - freeze.py: writes canonical pool-p1.toml (+ private counterpart, reusing draft's public/private split and private_digest) and freeze-p1.json (used-set digest, per-source counts, guard result, dropped count, label-check counts, freeze digest = sha256 of the canonical public manifest). A matching retry is a no-op; any changed frozen artifact requires a new pool id (never an overwrite). - protocol-p1.md: the frozen scoring/statistics protocol from design sections 3 and 4. Offline only; the guard subprocess is the sole subprocess and is mocked in tests. 42 new tests pass (incl the group-1.3 guard digest check on a fixture pool); neighbouring unseen suites green; ruff clean. New files only -- does not touch the journal-pinned draft/extract modules, so a running draw is unaffected. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
23ede07608
commit
197fe8a19c
@@ -0,0 +1,192 @@
|
||||
"""Offline guard controls: the hidden reservation exists only inside the fake runner."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
d = importlib.import_module("benchmarks.unseen.draft")
|
||||
b = importlib.import_module("benchmarks.unseen.bisect_guard")
|
||||
|
||||
|
||||
def draft_pair(tmp_path, count=9):
|
||||
root, cache = tmp_path / "tree", tmp_path / "cache"
|
||||
root.mkdir()
|
||||
cache.mkdir()
|
||||
rows = []
|
||||
for i in range(count):
|
||||
changes = [{"kind": "vulnerability", "method": "introducing", "base": "a" * 40, "head": "b" * 40}]
|
||||
changes[0]["label_check"] = ("confirmed", "refactor", "unclear")[i % 3]
|
||||
changes += [{"kind": "ordinary", "base": f"{j:040x}", "head": f"{j + 20:040x}"} for j in range(12)]
|
||||
rows.append(
|
||||
{
|
||||
"repository": f"synthetic/candidate-{i}",
|
||||
"url": "https://" + f"github.com/synthetic/candidate-{i}",
|
||||
"license_class": "private" if i % 2 else "permissive",
|
||||
"license_spdx": "GPL-3.0" if i % 2 else "MIT",
|
||||
"slice": i % 3 + 1,
|
||||
"changes": changes,
|
||||
}
|
||||
)
|
||||
rows[-1]["id"] = "r-" + d.private_digest(rows[-1])[:24]
|
||||
public, private = [], []
|
||||
for row in rows:
|
||||
if row["license_class"] == "private":
|
||||
private.append(row)
|
||||
public.append({"id": row["id"], "slice": row["slice"], "private": True, "sha256": d.private_digest(row)})
|
||||
else:
|
||||
public.append(row)
|
||||
draft = cache / "draft-p1.toml"
|
||||
draft.write_text(d.manifest(public, 19))
|
||||
(cache / "private-draft-p1.toml").write_text(d.manifest(private, 19))
|
||||
return root, draft, rows
|
||||
|
||||
|
||||
def staged(root):
|
||||
directory = root / "benchmarks/unseen"
|
||||
public = tomllib.loads((directory / "draft-p1.toml").read_text())
|
||||
private = tomllib.loads((directory / "private/draft-p1.toml").read_text())
|
||||
hidden = {row["id"]: row for row in private.get("repository", [])}
|
||||
rows = [hidden[row["id"]] if row.get("private") else row for row in public.get("repository", [])]
|
||||
assert set(hidden) == {row["id"] for row in public.get("repository", []) if row.get("private")}
|
||||
return rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("members", [(), (0,), (1, 4, 7), tuple(range(9))])
|
||||
def test_bisects_hidden_members_only(tmp_path, capsys, caplog, members):
|
||||
root, draft, rows = draft_pair(tmp_path)
|
||||
original = draft.read_bytes()
|
||||
reserved = {rows[i]["repository"] for i in members}
|
||||
calls = []
|
||||
|
||||
def guard(root):
|
||||
names = {row["repository"] for row in staged(root)}
|
||||
calls.append(names)
|
||||
return int(bool(names & reserved))
|
||||
|
||||
result = b.run(root, draft, guard_runner=guard)
|
||||
assert result["dropped_count"] == len(members)
|
||||
assert result["guard_result"] == 0
|
||||
assert {r["repository"] for r in staged(root)} == {r["repository"] for r in rows} - reserved
|
||||
assert calls[0] == set() and not (calls[-1] & reserved)
|
||||
assert draft.read_bytes() == original
|
||||
assert capsys.readouterr() == ("", "") and not caplog.records
|
||||
receipt = (root / "benchmarks/unseen/bisect-p1.json").read_text()
|
||||
for row in rows:
|
||||
assert row["repository"] not in receipt and row["url"] not in receipt
|
||||
assert json.loads(receipt) == result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [-9, 2, 3, 4, 5, 127])
|
||||
def test_unknown_exit_aborts_and_removes_staging(tmp_path, code):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
calls = 0
|
||||
|
||||
def guard(root):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return 0 if calls == 1 else code
|
||||
|
||||
with pytest.raises(b.InstrumentFailure, match="guard_exit"):
|
||||
b.run(root, draft, guard_runner=guard)
|
||||
assert not list((root / "benchmarks/unseen").rglob("*.toml"))
|
||||
assert not (root / "benchmarks/unseen/bisect-p1.json").exists()
|
||||
|
||||
|
||||
def test_red_empty_baseline_is_not_candidate_rejection(tmp_path):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
with pytest.raises(b.InstrumentFailure, match="guard_baseline"):
|
||||
b.run(root, draft, guard_runner=lambda root: 1)
|
||||
|
||||
|
||||
def test_final_combination_is_checked(tmp_path):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
codes = iter([0, 0, 1])
|
||||
with pytest.raises(b.InstrumentFailure, match="guard_final"):
|
||||
b.run(root, draft, guard_runner=lambda root: next(codes))
|
||||
|
||||
|
||||
def test_guard_subprocess_is_exact_and_discards_both_streams(tmp_path, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake(argv, **kwargs):
|
||||
calls.append((argv, kwargs))
|
||||
return subprocess.CompletedProcess(argv, 1)
|
||||
|
||||
monkeypatch.setattr(b.subprocess, "run", fake)
|
||||
assert b.pytest_guard(tmp_path) == 1
|
||||
argv, kwargs = calls[0]
|
||||
assert argv == [".venv/bin/pytest", "-q", "-p", "no:cacheprovider", "tests/test_independent_population.py", "-k", "referenced"]
|
||||
assert kwargs["cwd"] == tmp_path
|
||||
assert kwargs["stdout"] == kwargs["stderr"] == subprocess.DEVNULL
|
||||
|
||||
|
||||
def test_existing_staging_is_never_overwritten(tmp_path):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
path = root / "benchmarks/unseen/draft-p1.toml"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(b"preserve")
|
||||
with pytest.raises(FileExistsError):
|
||||
b.run(root, draft, guard_runner=lambda root: pytest.fail("guard should not run"))
|
||||
assert path.read_bytes() == b"preserve"
|
||||
|
||||
|
||||
def test_cli_outputs_dropped_count_only(tmp_path, monkeypatch, capsys):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
monkeypatch.setattr(b, "pytest_guard", lambda root: 0)
|
||||
assert b.main(["--root", str(root), "--draft", str(draft)]) == 0
|
||||
assert json.loads(capsys.readouterr().out) == {"dropped_count": 0}
|
||||
|
||||
|
||||
def test_cli_failure_does_not_echo_exception(tmp_path, monkeypatch, capsys):
|
||||
root, draft, rows = draft_pair(tmp_path)
|
||||
|
||||
def guard(root):
|
||||
raise OSError(rows[0]["url"])
|
||||
|
||||
monkeypatch.setattr(b, "pytest_guard", guard)
|
||||
assert b.main(["--root", str(root), "--draft", str(draft)]) == 1
|
||||
assert capsys.readouterr() == ('{"status": "instrument_failure"}\n', "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("problem", ["empty_file", "empty_pool", "missing_private", "bad_pointer", "private_in_public"])
|
||||
def test_invalid_draft_never_runs_guard(tmp_path, problem):
|
||||
root, draft, rows = draft_pair(tmp_path)
|
||||
private = draft.with_name("private-draft-p1.toml")
|
||||
if problem == "empty_file":
|
||||
draft.write_bytes(b"")
|
||||
elif problem == "empty_pool":
|
||||
draft.write_text(d.manifest([], 19))
|
||||
private.write_text(d.manifest([], 19))
|
||||
elif problem == "missing_private":
|
||||
private.unlink()
|
||||
elif problem == "bad_pointer":
|
||||
data = tomllib.loads(draft.read_text())
|
||||
data["repository"][1]["sha256"] = "0" * 64
|
||||
draft.write_text(d.manifest(data["repository"], 19))
|
||||
else:
|
||||
draft.write_text(d.manifest(rows, 19))
|
||||
with pytest.raises(b.InstrumentFailure):
|
||||
b.run(root, draft, guard_runner=lambda root: pytest.fail("guard should not run"))
|
||||
assert not (root / "benchmarks/unseen/draft-p1.toml").exists()
|
||||
|
||||
|
||||
def test_interrupt_removes_partial_subset(tmp_path):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
calls = 0
|
||||
|
||||
def guard(root):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls > 2:
|
||||
raise KeyboardInterrupt
|
||||
return 0 if calls == 1 else 1
|
||||
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
b.run(root, draft, guard_runner=guard)
|
||||
assert not list((root / "benchmarks/unseen").rglob("*.toml"))
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Freeze fixture pools without reading reservations or running subprocesses."""
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_unseen_bisect import b, d, draft_pair
|
||||
|
||||
f = importlib.import_module("benchmarks.unseen.freeze")
|
||||
|
||||
|
||||
def prepared(tmp_path):
|
||||
root, draft, rows = draft_pair(tmp_path)
|
||||
b.run(root, draft, guard_runner=lambda root: 0)
|
||||
protocol = root / "benchmarks/unseen/protocol-p1.md"
|
||||
protocol.write_bytes(Path("benchmarks/unseen/protocol-p1.md").read_bytes())
|
||||
return root, rows
|
||||
|
||||
|
||||
def used():
|
||||
return {"sha256": "a" * 64, "sources": {"pairs": {"count": 7}, "memory_s3": {"count": 3}}}
|
||||
|
||||
|
||||
def test_freeze_split_receipt_and_existing_guard_digest(tmp_path, capsys, caplog, monkeypatch):
|
||||
root, rows = prepared(tmp_path)
|
||||
record = f.run(root, used_report=used())
|
||||
directory = root / "benchmarks/unseen"
|
||||
pool = directory / "pool-p1.toml"
|
||||
public = tomllib.loads(pool.read_text())
|
||||
private = tomllib.loads((directory / "private/pool-p1.toml").read_text())
|
||||
assert public["status"] == private["status"] == "frozen"
|
||||
assert record["used_set_digest"] == "a" * 64
|
||||
assert record["source_counts"] == {"pairs": 7, "memory_s3": 3}
|
||||
assert record["label_check_counts"] == {"confirmed": 3, "refactor": 3, "unclear": 3, "unchecked": 0}
|
||||
assert record["guard_result"] == 0 and record["dropped_count"] == 0
|
||||
assert record["protocol_digest"] == hashlib.sha256((directory / "protocol-p1.md").read_bytes()).hexdigest()
|
||||
assert record["freeze_digest"] == hashlib.sha256(d.canonical(public)).hexdigest()
|
||||
assert json.loads((directory / "freeze-p1.json").read_text()) == record
|
||||
for row in rows:
|
||||
assert row["repository"] not in json.dumps(record) and row["url"] not in json.dumps(record)
|
||||
if row["license_class"] == "private":
|
||||
assert row["url"] not in pool.read_text() and row["repository"] not in pool.read_text()
|
||||
pointer = next(p for p in public["repository"] if p["id"] == row["id"])
|
||||
assert pointer == {"id": row["id"], "slice": row["slice"], "private": True, "sha256": d.private_digest(row)}
|
||||
assert row in private["repository"]
|
||||
else:
|
||||
assert row in public["repository"]
|
||||
assert (directory / "private/pool-p1.toml").stat().st_mode & 0o777 == 0o600
|
||||
guard = importlib.import_module("test_independent_population")
|
||||
monkeypatch.setattr(guard, "ROOT", root)
|
||||
guard.test_frozen_pool_manifest_digest()
|
||||
assert capsys.readouterr() == ("", "") and not caplog.records
|
||||
|
||||
|
||||
def test_digest_and_retry_ignore_toml_key_order(tmp_path):
|
||||
root, _ = prepared(tmp_path)
|
||||
original = f.run(root, used_report=used())
|
||||
pool = root / "benchmarks/unseen/pool-p1.toml"
|
||||
text = pool.read_text()
|
||||
lines = text.splitlines()
|
||||
pool.write_text("\n".join([lines[2], lines[1], lines[0], *lines[3:]]) + "\n")
|
||||
before = pool.read_bytes()
|
||||
assert f.run(root, used_report=used()) == original
|
||||
assert pool.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["pool-p1.toml", "private/pool-p1.toml", "freeze-p1.json"])
|
||||
def test_numeric_type_change_is_a_different_frozen_artifact(tmp_path, target):
|
||||
root, _ = prepared(tmp_path)
|
||||
f.run(root, used_report=used())
|
||||
path = root / "benchmarks/unseen" / target
|
||||
text = path.read_text()
|
||||
if path.suffix == ".toml":
|
||||
modified = text.replace("seed = 19", "seed = 19.0")
|
||||
else:
|
||||
modified = text.replace('"guard_result": 0', '"guard_result": false')
|
||||
assert text != modified
|
||||
path.write_text(modified)
|
||||
with pytest.raises(FileExistsError, match="new_pool"):
|
||||
f.run(root, used_report=used())
|
||||
assert path.read_text() == modified
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["pool-p1.toml", "private/pool-p1.toml", "freeze-p1.json", "protocol-p1.md"])
|
||||
def test_refuses_different_frozen_files(tmp_path, target):
|
||||
root, _ = prepared(tmp_path)
|
||||
f.run(root, used_report=used())
|
||||
directory = root / "benchmarks/unseen"
|
||||
path = directory / target
|
||||
if path.suffix == ".toml":
|
||||
path.write_text(path.read_text().replace('status = "frozen"', 'status = "modified"'))
|
||||
elif path.suffix == ".json":
|
||||
record = json.loads(path.read_text())
|
||||
record["dropped_count"] += 1
|
||||
path.write_text(json.dumps(record))
|
||||
else:
|
||||
path.write_text("changed protocol")
|
||||
before = {p: p.read_bytes() for p in directory.rglob("*") if p.is_file()}
|
||||
with pytest.raises(FileExistsError, match="new_pool"):
|
||||
f.run(root, used_report=used())
|
||||
assert all(p.read_bytes() == value for p, value in before.items())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["draft-p1.toml", "private/draft-p1.toml", "bisect-p1.json"])
|
||||
def test_refuses_stale_or_red_guard_receipt(tmp_path, target):
|
||||
root, _ = prepared(tmp_path)
|
||||
path = root / "benchmarks/unseen" / target
|
||||
if target.endswith(".json"):
|
||||
receipt = json.loads(path.read_text())
|
||||
receipt["guard_result"] = 1
|
||||
path.write_text(json.dumps(receipt))
|
||||
else:
|
||||
before = path.read_text()
|
||||
after = before.replace('"head" = "', '"head" = "0', 1)
|
||||
assert before != after
|
||||
path.write_text(after)
|
||||
with pytest.raises((ValueError, b.InstrumentFailure)):
|
||||
f.run(root, used_report=used())
|
||||
assert not (root / "benchmarks/unseen/pool-p1.toml").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["digest", "source_name", "count"])
|
||||
def test_rejects_identity_bearing_metadata(tmp_path, mutation):
|
||||
root, rows = prepared(tmp_path)
|
||||
report = used()
|
||||
if mutation == "digest":
|
||||
report["sha256"] = rows[0]["url"]
|
||||
elif mutation == "source_name":
|
||||
report["sources"][rows[0]["repository"]] = {"count": 1}
|
||||
else:
|
||||
report["sources"]["pairs"]["count"] = rows[0]["url"]
|
||||
with pytest.raises(ValueError):
|
||||
f.run(root, used_report=report)
|
||||
assert not (root / "benchmarks/unseen/freeze-p1.json").exists()
|
||||
|
||||
|
||||
def test_missing_protocol_aborts_before_publication(tmp_path):
|
||||
root, _ = prepared(tmp_path)
|
||||
(root / "benchmarks/unseen/protocol-p1.md").unlink()
|
||||
with pytest.raises(ValueError, match="protocol"):
|
||||
f.run(root, used_report=used())
|
||||
assert not (root / "benchmarks/unseen/pool-p1.toml").exists()
|
||||
|
||||
|
||||
def test_drops_flow_into_freeze_counts_and_private_manifest(tmp_path):
|
||||
root, draft, rows = draft_pair(tmp_path)
|
||||
hidden = {rows[i]["repository"] for i in (1, 2)}
|
||||
from test_unseen_bisect import staged
|
||||
|
||||
b.run(root, draft, guard_runner=lambda root: int(any(r["repository"] in hidden for r in staged(root))))
|
||||
directory = root / "benchmarks/unseen"
|
||||
(directory / "protocol-p1.md").write_bytes(Path("benchmarks/unseen/protocol-p1.md").read_bytes())
|
||||
record = f.run(root, used_report=used())
|
||||
assert record["dropped_count"] == 2
|
||||
assert record["label_check_counts"] == {"confirmed": 3, "refactor": 2, "unclear": 2, "unchecked": 0}
|
||||
assert len(tomllib.loads((directory / "pool-p1.toml").read_text())["repository"]) == 7
|
||||
assert len(tomllib.loads((directory / "private/pool-p1.toml").read_text())["repository"]) == 3
|
||||
|
||||
|
||||
def test_changed_private_labels_invalidate_guard_receipt(tmp_path):
|
||||
root, _ = prepared(tmp_path)
|
||||
path = root / "benchmarks/unseen/private/draft-p1.toml"
|
||||
before = path.read_text()
|
||||
after = before.replace('"label_check" = "refactor"', '"label_check" = "confirmed"', 1)
|
||||
assert before != after
|
||||
path.write_text(after)
|
||||
with pytest.raises(ValueError, match="guard_receipt"):
|
||||
f.run(root, used_report=used())
|
||||
|
||||
|
||||
def test_utf8_digest_matches_guard_with_non_ascii_metadata(tmp_path, monkeypatch):
|
||||
root, draft, _ = draft_pair(tmp_path)
|
||||
data = tomllib.loads(draft.read_text())
|
||||
data["repository"][0]["license_path"] = "licence-ä.txt"
|
||||
draft.write_text(d.manifest(data["repository"], 19))
|
||||
b.run(root, draft, guard_runner=lambda root: 0)
|
||||
(root / "benchmarks/unseen/protocol-p1.md").write_bytes(Path("benchmarks/unseen/protocol-p1.md").read_bytes())
|
||||
f.run(root, used_report=used())
|
||||
guard = importlib.import_module("test_independent_population")
|
||||
monkeypatch.setattr(guard, "ROOT", root)
|
||||
guard.test_frozen_pool_manifest_digest()
|
||||
|
||||
|
||||
def test_freeze_cli_redacts_failure_and_reports_only_record(tmp_path, capsys):
|
||||
root, rows = prepared(tmp_path)
|
||||
report = tmp_path / "used.json"
|
||||
report.write_text(json.dumps(used()))
|
||||
assert f.main(["--root", str(root), "--used-report", str(report)]) == 0
|
||||
output = capsys.readouterr()
|
||||
assert json.loads(output.out)["guard_result"] == 0 and output.err == ""
|
||||
assert all(r["repository"] not in output.out for r in rows)
|
||||
report.write_text(rows[0]["url"])
|
||||
assert f.main(["--root", str(root), "--used-report", str(report)]) == 1
|
||||
assert capsys.readouterr() == ('{"status": "freeze_refused"}\n', "")
|
||||
Reference in New Issue
Block a user