unseen: durable decision bank in ousast-corpus (stream + resume, retires the local journal)

The draw lost all progress on any restart because the local journal
invalidates on a code change and is the only cross-run state. Make the
bucket the source of truth: stream every per-repo accept/reject to
ousast-corpus as it is decided, keyed by sha256(repository+"\n"+fix) so
writes are idempotent (content dedup at rest), and resume by listing the
bucket and skipping already-decided repos. No supervisor/watchdog needed
-- a crash or a code change just re-runs and resumes.

New benchmarks/unseen/bank.py: corpus_settings (CORPUS_S3_* from .env,
missing key named, value never printed), CorpusObjects (lazy boto3,
path-style + SigV4 verified live against ousast-corpus @ 10.0.0.54:9001),
CorpusBank.bank/decided with one-retry put/get, split_decided. Decisions
live at <prefix>/<seed>/decisions/<key>.json with repository, fix,
accept/reject, reason, entry, exclusions, counts, contract, seed, ts.

draft.py: --bank [prefix] opens the bank on the coordinator (never shipped
to fork workers); run() resumes from bank.decided() before extracting,
reusing accepted entries across contracts (deterministic from repo+commit)
and rejections only when contract == EXTRACTION_CONTRACT ("v1", in
extract.py); collect() streams each decision one-by-one. Seed-order
acceptance, fork/used-set dedup, and duplicate-advisory reuse preserved.
Without --bank, local-journal behavior is unchanged.

CorpusBank proven live end-to-end (bank accept+reject, idempotent re-bank,
decided/split roundtrip). 325 tests pass offline, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

tests/test_unseen_bank.py

@@ -0,0 +1,216 @@
+"""Offline decision-bank contract: durability, identity and plain object transport."""
+
+import hashlib
+import io
+import json
+import sys
+from datetime import datetime
+from types import SimpleNamespace
+
+import pytest
+
+from benchmarks.unseen.bank import CorpusBank, CorpusObjects, corpus_settings, split_decided
+
+
+class FakeObjects:
+    def __init__(self):
+        self.objects = {}
+        self.labels = {}
+        self.calls = []
+        self.failures = {}
+
+    def call(self, operation):
+        self.calls.append(operation)
+        if self.failures.get(operation, 0):
+            self.failures[operation] -= 1
+            raise OSError("offline failure")
+
+    def get(self, key):
+        self.call("get")
+        return (self.objects[key], "version") if key in self.objects else None
+
+    def put(self, key, data, labels):
+        self.call("put")
+        self.objects[key] = data
+        self.labels[key] = labels
+
+    def keys(self, prefix):
+        return [key for key in reversed(self.objects) if key.startswith(prefix)]
+
+
+def settings_env():
+    return {
+        "CORPUS_S3_ENDPOINT": "https://objects.invalid",
+        "CORPUS_S3_BUCKET": "ousast-corpus",
+        "CORPUS_AWS_ACCESS_KEY_ID": "private-access",
+        "CORPUS_AWS_SECRET_ACCESS_KEY": "private-secret",
+    }
+
+
+@pytest.mark.parametrize("missing", list(settings_env()))
+def test_settings_names_missing_key_without_values(missing):
+    env = settings_env()
+    del env[missing]
+    with pytest.raises(ValueError) as error:
+        corpus_settings(env)
+    assert missing in str(error.value)
+    assert all(value not in str(error.value) for value in env.values())
+
+
+def test_settings_and_optional_region():
+    env = settings_env()
+    assert corpus_settings(env) == {
+        "endpoint": env["CORPUS_S3_ENDPOINT"],
+        "bucket": "ousast-corpus",
+        "access_key": "private-access",
+        "secret_key": "private-secret",
+    }
+    assert corpus_settings({**env, "CORPUS_S3_REGION": "region-1"})["region"] == "region-1"
+
+
+def test_bank_roundtrip_and_idempotent_bytes():
+    objects = FakeObjects()
+    bank = CorpusBank(objects, seed=42)
+    entry = {"repository": "org/a", "fix": "abc", "advisory": "GHSA-a", "changes": [{"code": "text"}]}
+    result = {"entry": entry, "ordinary_exclusions": {"merge": 2}, "counts": {"eligible": 13}}
+    key = bank.key_for("org/a", "abc")
+    assert key == hashlib.sha256(b"org/a\nabc").hexdigest()
+    bank.bank("org/a", "abc", result, contract="v1")
+    original = objects.objects.copy()
+    bank.bank("org/a", "abc", result, contract="v1")
+    assert objects.objects == original
+    assert list(objects.objects) == [f"unseen/42/decisions/{key}.json"]
+    row = bank.decided()[key]
+    assert row == {
+        "repository": "org/a",
+        "fix": "abc",
+        "decision": "accept",
+        "reason": None,
+        "entry": entry,
+        "ordinary_exclusions": {"merge": 2},
+        "counts": {"eligible": 13},
+        "contract": "v1",
+        "seed": 42,
+        "ts": row["ts"],
+    }
+    assert datetime.fromisoformat(row["ts"]).utcoffset().total_seconds() == 0
+    assert objects.labels[next(iter(objects.labels))] == {"decision": "accept", "contract": "v1"}
+
+
+def test_rejection_contract_update_and_split():
+    objects = FakeObjects()
+    bank = CorpusBank(objects, seed=3, prefix="custom")
+    for repo, advisory in (("z", "GHSA-z"), ("a", "GHSA-a")):
+        bank.bank(repo, "fix", {"entry": {"repository": repo, "fix": "fix", "advisory": advisory}}, contract="old")
+    bank.bank("no", "fix", {"rejected": "reason"}, contract="v1")
+    bank.bank("stale", "fix", {"rejected": "reason"}, contract="old")
+    accepted, keys = split_decided(bank.decided(), contract="v1")
+    assert [entry["repository"] for entry in accepted] == ["a", "z"]
+    assert keys == {bank.key_for(repo, "fix") for repo in ("a", "z", "no")}
+    row = bank.decided()[bank.key_for("no", "fix")]
+    assert row["reason"] == "reason" and row["decision"] == "reject"
+    assert row["entry"] is None and row["counts"] == row["ordinary_exclusions"] == {}
+    bank.bank("stale", "fix", {"rejected": "different"}, contract="v1")
+    assert len(objects.objects) == 4
+    assert bank.decided()[bank.key_for("stale", "fix")]["contract"] == "v1"
+
+
+@pytest.mark.parametrize("operation", ["get", "put"])
+@pytest.mark.parametrize("failures", [1, 2])
+def test_bank_retries_once_then_surfaces_failure(operation, failures):
+    objects = FakeObjects()
+    objects.failures[operation] = failures
+    bank = CorpusBank(objects, seed=1)
+    if failures == 1:
+        bank.bank("repo", "fix", {"rejected": "reason"}, contract="v1")
+        assert len(objects.objects) == 1
+    else:
+        with pytest.raises(OSError, match="offline failure"):
+            bank.bank("repo", "fix", {"rejected": "reason"}, contract="v1")
+    assert objects.calls.count(operation) == 2
+
+
+def test_decided_retries_get_and_fails_on_missing_listed_object():
+    objects = FakeObjects()
+    bank = CorpusBank(objects, seed=1)
+    bank.bank("repo", "fix", {"rejected": "reason"}, contract="v1")
+    objects.failures["get"] = 1
+    assert len(bank.decided()) == 1
+    objects.get = lambda key: None
+    with pytest.raises(ValueError, match="missing"):
+        bank.decided()
+
+
+def test_plain_s3_adapter_without_bucket_probes(monkeypatch):
+    calls = []
+
+    class Missing(Exception):
+        response = {"Error": {"Code": "NoSuchKey"}}
+
+    class S3:
+        def put_object(self, **kwargs):
+            calls.append(("put", kwargs))
+
+        def get_object(self, **kwargs):
+            calls.append(("get", kwargs))
+            if kwargs["Key"] == "missing":
+                raise Missing()
+            return {"Body": io.BytesIO(b"data"), "ETag": "tag"}
+
+        def get_paginator(self, name):
+            assert name == "list_objects_v2"
+            return SimpleNamespace(paginate=self.pages)
+
+        def pages(self, **kwargs):
+            calls.append(("list", kwargs))
+            return [{"Contents": [{"Key": "p/a"}]}, {}, {"Contents": [{"Key": "p/b"}]}]
+
+    def client(service, **kwargs):
+        calls.append((service, kwargs))
+        return S3()
+
+    monkeypatch.setitem(sys.modules, "boto3", SimpleNamespace(client=client))
+    monkeypatch.setitem(sys.modules, "botocore.config", SimpleNamespace(Config=lambda **kwargs: SimpleNamespace(**kwargs)))
+    adapter = CorpusObjects(**corpus_settings(settings_env()))
+    adapter.put("p/a", b"data", {"decision": "accept", "contract": "v1"})
+    assert adapter.get("p/a") == (b"data", "tag")
+    assert adapter.get("missing") is None
+    assert adapter.keys("p/") == ["p/a", "p/b"]
+    assert calls[0][0] == "s3"
+    assert calls[0][1]["endpoint_url"] == "https://objects.invalid"
+    assert calls[0][1]["config"].retries == {"total_max_attempts": 1}
+    assert calls[1][1]["Bucket"] == "ousast-corpus"
+    assert calls[1][1]["Body"] == b"data"
+    assert calls[-1] == ("list", {"Bucket": "ousast-corpus", "Prefix": "p/"})
+
+
+def test_decided_rejects_invalid_identity():
+    objects = FakeObjects()
+    bank = CorpusBank(objects, seed=1)
+    bank.bank("repo", "fix", {"rejected": "reason"}, contract="v1")
+    key = next(iter(objects.objects))
+    row = json.loads(objects.objects[key])
+    row["repository"] = "other"
+    objects.objects[key] = json.dumps(row).encode()
+    with pytest.raises(ValueError, match="identity"):
+        bank.decided()
+
+
+@pytest.mark.parametrize("endpoint", ["missing-scheme-private", "https://private[", "http://private／secret"])
+def test_invalid_endpoint_does_not_expose_value(endpoint):
+    with pytest.raises(ValueError) as error:
+        corpus_settings({**settings_env(), "CORPUS_S3_ENDPOINT": endpoint})
+    assert "CORPUS_S3_ENDPOINT" in str(error.value)
+    assert endpoint not in str(error.value)
+
+
+def test_import_does_not_require_boto3(monkeypatch):
+    import importlib.util
+
+    from benchmarks.unseen import bank
+
+    monkeypatch.setitem(sys.modules, "boto3", None)
+    spec = importlib.util.spec_from_file_location("offline_bank", bank.__file__)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    assert module.CorpusBank.key_for("a", "b") == CorpusBank.key_for("a", "b")

tests/test_unseen_draft.py

@@ -1519,3 +1519,301 @@ def test_pool_configuration_change_refuses_resume_before_io(tmp_path, changed):
     )
     assert report["status"] == "resume_refused" and report["changed_parameters"] == list(changed)
     assert api.calls == 0 and not clones.counts
+
+
+class CorpusFakeObjects:
+    def __init__(self):
+        self.objects = {}
+        self.coordinator = os.getpid()
+
+    def get(self, key):
+        assert os.getpid() == self.coordinator
+        return (self.objects[key], "v1") if key in self.objects else None
+
+    def put(self, key, data, labels):
+        assert os.getpid() == self.coordinator
+        self.objects[key] = data
+
+    def keys(self, prefix):
+        assert os.getpid() == self.coordinator
+        return list(reversed([key for key in self.objects if key.startswith(prefix)]))
+
+
+@pytest.mark.parametrize("workers", [1, 4])
+@pytest.mark.parametrize("short_refs", [False, True])
+def test_bank_resume_stream_contract_seed_order_and_forks(tmp_path, monkeypatch, workers, short_refs):
+    from benchmarks.unseen.bank import CorpusBank
+
+    monkeypatch.setattr(importlib.import_module("benchmarks.unseen.journal").os, "fsync", lambda fd: None)
+    ordered = s.ordered_advisories(list(FakeAPI(10).advisories()), 7)
+    names = [next(iter(s.fix_links(row)))[0] for row in ordered]
+    ranks = {name: i for i, name in enumerate(names)}
+    called = multiprocessing.get_context("fork").Array("i", 10)
+    changes = entries()[0]["changes"]
+    selected = []
+    original_assign = d.assign_slices
+
+    class API(FakeAPI):
+        def advisories(self):
+            for row in super().advisories():
+                if short_refs:
+                    row["references"] = [row["references"][0][:-33]]
+                yield row
+
+        def get_commit(self, name, sha):
+            return super().get_commit(name, "a" * 40)
+
+        def get_repo(self, name):
+            result = super().get_repo(name)
+            if name == names[3]:
+                result["source"] = {"full_name": names[1]}
+            return result
+
+    def extract_changes(git, candidate, fixes, *, seed):
+        rank = ranks[candidate.repository]
+        called[rank] += 1
+        if rank == 0:
+            raise s.Rejected("no_ordinary")
+        return copy.deepcopy(changes), {"eligible": 40}, {"bytes": 12, "files": 1}
+
+    def assign(rows, **kwargs):
+        selected.append([row["repository"] for row in rows])
+        return original_assign(rows, **kwargs)
+
+    monkeypatch.setattr(d, "repository_changes", extract_changes)
+    monkeypatch.setattr(d, "assign_slices", assign)
+
+    def run(path, bank=None):
+        used = e.UsedSet()
+        return d.run(
+            tmp_path / "repo",
+            github=API(10),
+            osv=FakeOSV(),
+            clones=FakeClones(),
+            used=used,
+            seed=7,
+            known_empty=set(used.sources),
+            journal_path=path,
+            extract_workers=workers,
+            pool_size=4,
+            slices=1,
+            bank=bank,
+        )
+
+    full_objects = CorpusFakeObjects()
+    full_bank = CorpusBank(full_objects, seed=7)
+    full = run(tmp_path / "full", full_bank)
+    assert full["status"] == "draft", full
+    decisions = full_bank.decided()
+    assert len(decisions) == sum(called)
+    assert {obj["decision"] for obj in decisions.values()} == {"accept", "reject"}
+    resumed_objects = CorpusFakeObjects()
+    resumed_bank = CorpusBank(resumed_objects, seed=7)
+    # Partial completion can be out of seed order, including a fork that loses
+    # to an earlier fresh result. Rejections alone are contract sensitive.
+    for obj in decisions.values():
+        rank = ranks[obj["repository"]]
+        if rank in {0, 2, 3}:
+            result = {"entry": obj["entry"]} if obj["decision"] == "accept" else {"rejected": obj["reason"]}
+            result.update(ordinary_exclusions=obj["ordinary_exclusions"], counts=obj["counts"])
+            resumed_bank.bank(
+                obj["repository"], obj["fix"], result, contract=d.EXTRACTION_CONTRACT if rank == 0 else "old-accepted-contract"
+            )
+    resumed_bank.bank(names[1], "a" * 40, {"rejected": "old_failure"}, contract="old")
+    before = list(called)
+    resumed = run(tmp_path / "resume", resumed_bank)
+    assert resumed["status"] == "draft", resumed
+    assert selected == [[names[i] for i in (1, 2, 4, 5)]] * 2
+    assert resumed["manifest_sha256"] == full["manifest_sha256"]
+    assert all(called[i] == before[i] for i in (0, 2, 3))
+    assert called[1] == before[1] + 1
+    for filename in ("draft-p1.toml", "private-draft-p1.toml"):
+        assert (tmp_path / "resume" / filename).read_bytes() == (tmp_path / "full" / filename).read_bytes()
+    # A corrupt/stale local journal and code fingerprint must not be cross-run truth.
+    (tmp_path / "resume" / "inputs.json").write_text('{"design_draft":"old"}')
+    (tmp_path / "resume" / "extraction.jsonl").write_text("torn")
+    before = list(called)
+    again = run(tmp_path / "resume", resumed_bank)
+    assert again["manifest_sha256"] == full["manifest_sha256"]
+    assert list(called) == before
+
+
+@pytest.mark.parametrize("failure", ["crash", "persistent_put"])
+def test_bank_crash_resume_and_durability_failure(tmp_path, monkeypatch, failure):
+    from benchmarks.unseen.bank import CorpusBank
+
+    monkeypatch.setattr(importlib.import_module("benchmarks.unseen.journal").os, "fsync", lambda fd: None)
+    changes = entries()[0]["changes"]
+    monkeypatch.setattr(d, "repository_changes", lambda *a, **kw: (copy.deepcopy(changes), {}, {"bytes": 1, "files": 1}))
+
+    class Objects(CorpusFakeObjects):
+        interrupted = False
+        puts = 0
+
+        def put(self, key, data, labels):
+            self.puts += 1
+            if failure == "persistent_put" and not self.interrupted:
+                raise OSError("offline simulated bank outage")
+            super().put(key, data, labels)
+            if not self.interrupted:
+                self.interrupted = True
+                raise KeyboardInterrupt
+
+    objects = Objects()
+    bank = CorpusBank(objects, seed=7)
+
+    def run(path, bank):
+        used = e.UsedSet()
+        return d.run(
+            tmp_path / "repo",
+            github=FakeAPI(8),
+            osv=FakeOSV(),
+            clones=FakeClones(),
+            used=used,
+            seed=7,
+            known_empty=set(used.sources),
+            journal_path=path,
+            extract_workers=1,
+            pool_size=3,
+            slices=1,
+            bank=bank,
+        )
+
+    if failure == "crash":
+        with pytest.raises(KeyboardInterrupt):
+            run(tmp_path / "resume", bank)
+        assert len(bank.decided()) == 1
+    else:
+        report = run(tmp_path / "resume", bank)
+        assert report["status"] == "instrument_failure"
+        assert objects.puts == 2
+        assert not objects.objects
+        assert not (tmp_path / "resume" / "draft-p1.toml").exists()
+        objects.interrupted = True
+    resumed = run(tmp_path / "resume", bank)
+    full = run(tmp_path / "full", None)
+    assert resumed["status"] == full["status"] == "draft"
+    assert resumed["manifest_sha256"] == full["manifest_sha256"]
+
+
+@pytest.mark.parametrize("bank_args,prefix", [([], None), (["--bank"], "unseen"), (["--bank", "custom"], "custom")])
+def test_cli_corpus_bank_is_opt_in(tmp_path, monkeypatch, capsys, bank_args, prefix):
+    from openultrasast import config
+    from openultrasast.plane import memory
+
+    objects = CorpusFakeObjects()
+    monkeypatch.setattr(config, "load_dotenv", lambda *a: None)
+    monkeypatch.setattr(memory, "open_store", lambda *a, **kw: object())
+    monkeypatch.setattr(d.eligibility, "build_used", lambda *a, **kw: e.UsedSet())
+    monkeypatch.setattr(d.source, "GitHub", lambda *a: FakeAPI(0))
+    monkeypatch.setattr(d.source, "OSV", FakeOSV)
+    settings_calls = []
+    monkeypatch.setattr(d, "corpus_settings", lambda: settings_calls.append(True) or {"bucket": "ousast-corpus"})
+
+    def make_objects(**kwargs):
+        assert kwargs == {"bucket": "ousast-corpus"}
+        return objects
+
+    def run(*args, **kwargs):
+        bank = kwargs["bank"]
+        if prefix is None:
+            assert bank is None
+        else:
+            assert bank.client is objects and bank.decisions_prefix == f"{prefix}/7/decisions/"
+        return {"status": "draft"}
+
+    monkeypatch.setattr(d, "CorpusObjects", make_objects)
+    monkeypatch.setattr(d, "run", run)
+    assert d.main(["--root", str(tmp_path / "repo"), "--cache", str(tmp_path / "cache"), "--seed", "7", *bank_args]) == 0
+    assert settings_calls == ([] if prefix is None else [True])
+
+
+def test_bank_accepts_survive_missing_advisories_and_obey_used_networks(tmp_path, monkeypatch):
+    from benchmarks.unseen.bank import CorpusBank
+
+    objects = CorpusFakeObjects()
+    bank = CorpusBank(objects, seed=7)
+    rows = s.ordered_advisories(list(FakeAPI(4).advisories()), 7)
+    names = [next(iter(s.fix_links(row)))[0] for row in rows]
+    for row in rows:
+        candidate = s.candidate(row, FakeAPI(), FakeOSV())
+        entry = {**d.asdict(candidate), "changes": entries()[0]["changes"], "instrument": {"bytes": 12, "files": 1}}
+        bank.bank(candidate.repository, candidate.fix, {"entry": entry}, contract="older")
+
+    class API(FakeAPI):
+        def get_repo(self, name):
+            result = super().get_repo(name)
+            if name == names[2]:
+                result["source"] = {"full_name": names[1]}
+            return result
+
+        def get_commit(self, *args):
+            pytest.fail("banked accepts must not be re-decided")
+
+    used = e.UsedSet()
+    used.add("pairs", {names[0]}, size=len(names[0]))
+    selected = []
+    original = d.assign_slices
+
+    def assign(rows, **kwargs):
+        selected.extend(row["repository"] for row in rows)
+        return original(rows, **kwargs)
+
+    monkeypatch.setattr(d, "assign_slices", assign)
+    report = d.run(
+        tmp_path / "repo",
+        github=API(0),
+        osv=FakeOSV(),
+        clones=FakeClones(),
+        used=used,
+        seed=7,
+        known_empty={key for key, names in used.sources.items() if not names},
+        journal_path=tmp_path / "journal",
+        pool_size=2,
+        slices=1,
+        bank=bank,
+    )
+    assert report["status"] == "draft", report
+    assert selected == [names[1], names[3]]
+    assert report["rejections"]["fork_network"] == 1
+    assert len(bank.decided()) == 4
+
+
+@pytest.mark.parametrize("workers", [1, 4])
+def test_bank_duplicate_advisory_cannot_overwrite_pair_decision(tmp_path, monkeypatch, workers):
+    from benchmarks.unseen.bank import CorpusBank
+
+    rows = s.ordered_advisories(list(FakeAPI(6).advisories()), 7)
+    rows[1]["references"] = rows[0]["references"][:]
+    rows[1]["cwes"] = []  # Would reject if re-decided under this later advisory.
+    repository, fix = next(iter(s.fix_links(rows[0])))
+
+    class API(FakeAPI):
+        def advisories(self):
+            return iter(copy.deepcopy(rows))
+
+    changes = entries()[0]["changes"]
+    monkeypatch.setattr(d, "repository_changes", lambda *a, **kw: (copy.deepcopy(changes), {}, {"bytes": 12, "files": 1}))
+    bank = CorpusBank(CorpusFakeObjects(), seed=7)
+    reports = []
+    for name in ("first", "resumed"):
+        used = e.UsedSet()
+        reports.append(
+            d.run(
+                tmp_path / "repo",
+                github=API(),
+                osv=FakeOSV(),
+                clones=FakeClones(),
+                used=used,
+                known_empty=set(used.sources),
+                seed=7,
+                pool_size=3,
+                slices=1,
+                extract_workers=workers,
+                journal_path=tmp_path / name,
+                bank=bank,
+            )
+        )
+    assert all(report["status"] == "draft" for report in reports)
+    assert reports[0]["manifest_sha256"] == reports[1]["manifest_sha256"]
+    assert bank.decided()[bank.key_for(repository, fix)]["decision"] == "accept"