unseen-eval group 2 (2.1-2.4): advisory sourcing, vulnerability-introducing ranges, ordinary-commit draw, draft manifest and slices; journal with resume

Implemented with Codex. Sourcing run (2.5) next, by the coordinator.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/unseen/journal.py

@@ -0,0 +1,184 @@
+"""Private, append-only sourcing checkpoints. No identity-bearing data leaves this directory."""
+
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import json
+import math
+import os
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+STAGES = ("advisories", "resolution", "eligibility", "extraction", "completion")
+
+
+def encoded(value) -> bytes:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
+
+
+def digest(value) -> str:
+    return hashlib.sha256(encoded(value)).hexdigest()
+
+
+class ResumeMismatch(ValueError):
+    def __init__(self, parameters):
+        self.parameters = parameters
+        super().__init__("resume_inputs_changed")
+
+
+class Stopped(Exception):
+    pass
+
+
+def external(root: Path, path: Path) -> Path:
+    path = path.expanduser().resolve()
+    # Also reject the actual checkout when a different --root is supplied.
+    if any(path.is_relative_to(p.resolve()) for p in (root, Path(__file__).resolve().parents[2])):
+        raise ValueError("cache_inside_repository")
+    return path
+
+
+def atomic(path: Path, payload: bytes) -> None:
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    with temporary.open("wb") as handle:
+        os.chmod(temporary, 0o600)
+        handle.write(payload)
+        handle.flush()
+        os.fsync(handle.fileno())
+    temporary.replace(path)
+    sync_directory(path.parent)
+
+
+def sync_directory(path: Path) -> None:
+    fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+class Journal:
+    def __init__(self, path: Path, *, max_hours=None, clock=None, rate_limit=None):
+        if max_hours is not None and (not math.isfinite(max_hours) or max_hours < 0):
+            raise ValueError("invalid_max_hours")
+        self.path = path
+        self.clock = clock or time.monotonic
+        self.started = self.clock()
+        self.max_seconds = None if max_hours is None else max_hours * 3600
+        self.rate_limit = rate_limit or (lambda: {})
+        self.rows = {stage: {} for stage in STAGES}
+        self.elapsed_before = 0.0
+        self.last = None
+        self.previous_rate = {}
+
+    @contextmanager
+    def open(self, parameters):
+        self.path.mkdir(parents=True, exist_ok=True, mode=0o700)
+        with (self.path / "LOCK").open("a") as lock:
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                raise ValueError("journal_busy") from None
+            metadata = self.path / "inputs.json"
+            if metadata.exists():
+                old = json.loads(metadata.read_bytes())
+                # Only current, fixed parameter labels can enter stdout.
+                changed = [key for key in parameters if old.get(key) != parameters[key]]
+                if old.keys() != parameters.keys():
+                    changed.append("parameter_schema")
+                if changed:
+                    raise ResumeMismatch(changed)
+            else:
+                if any(self.path.glob("*.jsonl")):
+                    raise ValueError("journal_missing_inputs")
+                atomic(metadata, encoded(parameters))
+            for stage in STAGES:
+                path = self.path / (stage + ".jsonl")
+                if not path.exists():
+                    continue
+                with path.open("rb") as handle:
+                    for line in handle:
+                        # Fail closed on torn/corrupt writes; never silently lose a completed unit.
+                        if not line.endswith(b"\n"):
+                            raise ValueError("journal_incomplete_record")
+                        row = json.loads(line)
+                        if row["id"] in self.rows[stage]:
+                            raise ValueError("journal_duplicate_unit")
+                        self.rows[stage][row["id"]] = row["value"]
+                        if row["elapsed_seconds"] >= self.elapsed_before:
+                            self.elapsed_before = row["elapsed_seconds"]
+                            self.last = {"stage": stage, "id": row["id"]}
+                            self.previous_rate = row["rate_limit"]
+            self.progress("sourcing")
+            yield self
+
+    def boundary(self):
+        reason = None
+        if (self.path / "STOP").exists():
+            reason = "STOP"
+        elif self.max_seconds is not None and self.clock() - self.started >= self.max_seconds:
+            reason = "max_hours"
+        if reason:
+            self.progress("stopped", reason=reason)
+            raise Stopped(reason)
+
+    def elapsed(self):
+        return self.elapsed_before + self.clock() - self.started
+
+    def progress(self, status, **extra):
+        atomic(
+            self.path / "progress.json",
+            encoded(
+                {
+                    "status": status,
+                    "counts": {stage: len(rows) for stage, rows in self.rows.items()},
+                    "last_unit": self.last,
+                    "rate_limit": self.rate_limit() or self.previous_rate,
+                    "elapsed_seconds": self.elapsed(),
+                    **extra,
+                }
+            ),
+        )
+
+    def put(self, stage, key, value):
+        if key in self.rows[stage]:
+            raise ValueError("journal_duplicate_unit")
+        row = {"id": key, "value": value, "elapsed_seconds": self.elapsed(), "rate_limit": self.rate_limit() or self.previous_rate}
+        path = self.path / (stage + ".jsonl")
+        with path.open("ab") as handle:
+            os.chmod(path, 0o600)
+            handle.write(encoded(row) + b"\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        sync_directory(self.path)
+        self.rows[stage][key] = value
+        self.last = {"stage": stage, "id": key}
+        self.progress("sourcing")
+
+
+class ResolvedGitHub:
+    """Persist successful repository lookups and 404s used by both eligibility and the fix index."""
+
+    def __init__(self, github, journal):
+        self.github, self.journal = github, journal
+
+    def __getattr__(self, name):
+        return getattr(self.github, name)
+
+    def get_repo(self, name):
+        from .eligibility import RepositoryNotFound, normalize
+
+        key = normalize(name)
+        if key not in self.journal.rows["resolution"]:
+            self.journal.boundary()
+            try:
+                value = self.github.get_repo(key)
+            except RepositoryNotFound:
+                value = None
+            self.journal.put("resolution", key, value)
+        value = self.journal.rows["resolution"][key]
+        if value is None:
+            raise RepositoryNotFound("not_found")
+        return value