mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 04:59:28 +00:00
plane(2.4): reconciler on Delivery -- the start body carries the URLs, the store completes the task
_execute builds the task's `delivery {put, inputs}` from Delivery, passes it to
router.starter, grants the store's host in the egress policy, awaits
Delivery.delivered (the tar's exact key in the store), then collect()s:
download, extract with the member checks, re-put the declared artifacts, write
done.json; the outcome is the collected summary.json. Three findings from the
tests: a rerun of an unfinished task completed at once on the previous
attempt's tar, so Delivery.reset(task) deletes output.tar and done.json before
every attempt; a worker thread that raised left the run blocked on its queue,
so the worker now turns any exception into a failed task; a consumer may
declare a producer's summary.json without the producer listing it (the loop
Run does), so declared(run) publishes outputs[] plus every artifact a consumer
names. The receiver thread survives behind OUSAST_DELIVERY=receiver for this
group only (no presigned URLs, the http rule, OUSAST_ARTIFACT_URL in the env);
the default path renders no receiver address into any Task. The fake ax PUTs
to the start body's URL and GETs its inputs from theirs; the reconciler tests
run through the S3-shaped fake store and assert the objects, done.json, the
seeded rerun and the receiver mode. reconciler.py: 549 lines (bound 550).
Gates: pytest 2031 passed, 42 skipped, exit 0; ruff check 0; ruff format 0;
mypy 0.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
c49eae4f30
commit
605b7af5ba
@@ -31,9 +31,10 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .manifests import Run
|
||||
from .memory import RUNS_PREFIX, MemoryStore
|
||||
|
||||
__all__ = ["DONE", "GRACE_SECONDS", "OUTPUT_TAR", "STATE", "Delivery", "DeliveryError", "expiry", "input_var"]
|
||||
__all__ = ["DONE", "GRACE_SECONDS", "OUTPUT_TAR", "STATE", "Delivery", "DeliveryError", "declared", "expiry", "input_var"]
|
||||
|
||||
OUTPUT_TAR = "output.tar"
|
||||
DONE = "done.json"
|
||||
@@ -62,6 +63,18 @@ def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def declared(run: Run) -> dict[str, list[str]]:
|
||||
"""Per Run task, the artifacts to publish as single objects: its ``outputs[]`` plus every ``<producer>/<artifact>``
|
||||
a consumer declares as an input on it (``summary.json`` of a producer is a usual one), in declaration order."""
|
||||
wanted: dict[str, list[str]] = {entry.name: list(entry.outputs) for entry in run.tasks}
|
||||
for entry in run.tasks:
|
||||
for ref in entry.inputs.values():
|
||||
producer, _, artifact = ref.partition("/")
|
||||
if producer in wanted and artifact and artifact not in wanted[producer]:
|
||||
wanted[producer].append(artifact)
|
||||
return wanted
|
||||
|
||||
|
||||
class Delivery:
|
||||
"""Presigned URLs, polling, collection and state mirroring for one Run on one store."""
|
||||
|
||||
@@ -98,6 +111,12 @@ class Delivery:
|
||||
|
||||
# --- completion ----------------------------------------------------------------------------------------------
|
||||
|
||||
def reset(self, task: str) -> None:
|
||||
"""Before an attempt: remove the previous attempt's tar and ``done.json`` so a rerun (an unfinished task
|
||||
resumed with a larger budget, an interrupted attempt) never completes on a stale object."""
|
||||
for name in (OUTPUT_TAR, DONE):
|
||||
self.store._delete(self.key(task, name))
|
||||
|
||||
def delivered(self, task: str) -> bool:
|
||||
"""Whether the runner's tar is in the store (an exact-key listing, the store's HEAD)."""
|
||||
key = self.key(task, OUTPUT_TAR)
|
||||
|
||||
Reference in New Issue
Block a user