feat(plane): the improvement loop as a plane Run (harnessx-removal task 4)

`ousast plane workspaces --validation-set ... --loop` appends the loop to a Run:
per case `alerts` (plane/tasks/alerts.py: the quick-mode scan on the vulnerable
and the fixed pin, a fixed-pin Workspace per case, alerts attributed to their
enclosing function by the repo-facts patterns, `in_fix_range` from the fix's
old-side and new-side ranges, failing on an unread tree) feeding `remember`,
then `memory-snapshot` -> `loop-measure` -> `loop-propose` -> `loop-improve`
(plane/tasks/loop.py). Every new task binds no Model and has budget
{usd: 0, calls: 0}. The snapshot is the store's rows after the train-on-test
guard, written on the host by memory.seed (a sandboxed run of it fails loudly);
propose runs M1/M2 against this repository at the commit under test; improve
runs one unchanged evolve.run_round and delivers gate.json, journal.json, the
sidecar, rule_policy.json only when accepted, and proposal_outcome rows that
the host ingests. Adoption stays a maintainer commit.

`remember` is now a Run entry per case (the image contains it), with the facts
row keyed by the image the facts were computed on (repo-facts records it in
its summary; unknown -> the row is omitted, never guessed). Alert rows carry
their own pin; improve/memory.py counts a repeated alert once.

Runner image rebuilt (+0.39 MB: only the new modules; the quick scan needs no
extra and the rulesets were already in the package), pushed, plane/ re-pinned
and regenerated (validation-46: 105 entries). Fixture test runs the whole loop
graph through the fake ax, whose new `exec` mode runs the real task modules.
Live: one loop Run over three validation-46 cases on the local ax, recorded in
benchmarks/measurements/2026-09-30-harnessx-removal-loop/ (all done, $0 new
spend, 0 proposals, gate verdict no_proposals).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 11:59:50 +02:00
co-authored by Claude Opus 5.5
parent d104166697
commit b68fe27c74
29 changed files with 2651 additions and 140 deletions
+22 -3
View File
@@ -284,6 +284,16 @@ def main(argv: list[str] | None = None) -> int:
plane_ws.add_argument(
"--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)"
)
plane_ws.add_argument(
"--loop", action="store_true",
help="with --validation-set: append the improvement loop (alerts per case, then snapshot, measure, propose, improve)",
) # fmt: skip
plane_ws.add_argument("--loop-commit", help="with --loop: the commit under test the gate runs at (default: git rev-parse HEAD)")
plane_ws.add_argument("--loop-repo", help="with --loop: this repository's URL (default: git remote get-url origin)")
plane_ws.add_argument("--loop-manifest", help="with --loop: the benchmark manifest the proposals are gated on")
plane_ws.add_argument(
"--qualify-population", action="append", default=[], metavar="NAME", help="with --loop: a population whose rows never propose"
)
args = parser.parse_args(argv)
# Diagnosis needs the stage costs, and nothing configures logging, so the default root level of
@@ -1382,7 +1392,7 @@ def _plane_memory(
assert run_manifest is not None
seeded = memory.seed(run_manifest, store)
if seeded:
print(f"memory {store.describe()}: facts reused for {len(seeded)} tasks: {', '.join(seeded)}")
print(f"memory {store.describe()}: seeded {len(seeded)} tasks (reused facts, loop snapshot): {', '.join(seeded)}")
return 0
annotations: dict[str, str] = {}
if run_manifest is not None:
@@ -1403,7 +1413,7 @@ def _plane_memory(
def _plane_increment(args: argparse.Namespace) -> int:
from .plane.generate import increment
from .plane.generate import LOOP_MANIFEST, Loop, increment
if args.candidates is None or args.triage is None:
print("--validation-set needs --candidates and --triage", file=sys.stderr)
@@ -1417,9 +1427,18 @@ def _plane_increment(args: argparse.Namespace) -> int:
)
if args.runner_image is not None:
command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}"
loop = None
if args.loop:
git = lambda *a: subprocess.run(["git", *a], capture_output=True, text=True, check=True).stdout.strip() # noqa: E731
loop = Loop(
repo=args.loop_repo or git("remote", "get-url", "origin"), commit=args.loop_commit or git("rev-parse", "HEAD"),
manifest=args.loop_manifest or LOOP_MANIFEST, populations=tuple(args.qualify_population),
) # fmt: skip
command += f" --loop --loop-commit {loop.commit} --loop-repo {loop.repo} --loop-manifest {loop.manifest}"
command += "".join(f" --qualify-population {p}" for p in loop.populations)
written = increment(
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
repos=args.repos, runner_image=args.runner_image,
repos=args.repos, runner_image=args.runner_image, loop=loop,
) # fmt: skip
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
return 0
+7 -1
View File
@@ -145,7 +145,10 @@ class _View:
@classmethod
def of(cls, records: Sequence[Record]) -> _View:
"""One alert per (repo, pin, rule, path, line, function, pin role): every loop run re-alerts an unchanged pin under a
new run name, and a repeat is the same evidence, not more of it."""
view = cls()
seen: set[tuple[object, ...]] = set()
for rec in records:
row = rec.row
if row["kind"] == "verdict":
@@ -153,7 +156,10 @@ class _View:
if row.get("site_match") is True:
view.sites.add((row["repo"], str(row.get("candidate"))))
elif row["kind"] == "alert" and row.get("rule_id"):
view.alerts.setdefault(str(row["rule_id"]), []).append(rec)
place = (row["repo"], row["pin"], row["rule_id"], *(row.get(k) for k in ("path", "line", "function", "pin_role")))
if place not in seen:
seen.add(place)
view.alerts.setdefault(str(row["rule_id"]), []).append(rec)
return view
def verdict(self, alert: Mapping[str, Any]) -> Record | None:
+2
View File
@@ -65,6 +65,8 @@ SPEC_OWNED = {
"plane.generate": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set",
"plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`",
"plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free",
"plane.tasks.alerts": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free",
"plane.tasks.loop": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps",
"plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`",
"plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory",
}
+156 -18
View File
@@ -15,8 +15,16 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
(``triage.json``, for the measurement);
- ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
Workspaces and naming the files as env;
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets, annotated with the
population and split (``openultrasast.io/population``/``split``: the files' stems) the memory rows carry.
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree -> verify c -> final -> remember``,
with budgets, annotated with the population and split (``openultrasast.io/population``/``split``: the files'
stems) the memory rows carry.
``--loop`` (harnessx-removal design section 6) appends the improvement loop: per case ``alerts`` (quick-mode rules
on the vulnerable and the fixed pin, which gets its Workspace ``<case>-fixed``; ``case.json`` then also carries the
fix's new-side ranges) feeding ``remember``, and once ``memory-snapshot`` (the store's rows after the train-on-test
guard, written on the host by ``memory.seed``) -> ``loop-measure`` -> ``loop-propose`` -> ``loop-improve``, the last
two bound to this repository at the commit under test. All of them model-free with budget ``{usd: 0, calls: 0}``;
the gate's verdict is ``<run>/loop-improve/gate.json``, and adopting an accepted ledger stays a maintainer commit.
Each ``repo-facts`` Task carries ``openultrasast.io/memory-key`` (repository, pin, candidates digest, runner image
digest): the key under which stored facts are reused instead of recomputed (``plane/memory.py``, ``seed``).
@@ -57,12 +65,12 @@ import yaml
from ..model.endpoint import price_of
from .budget import cost_of
from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests
from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, memory_key
from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SNAPSHOT_ANNOTATION, SPLIT_ANNOTATION, memory_key
from .reconciler import _ax_name, _doc
from .tasks.repo_facts import _digest
from .tasks.verify import MAX_STEPS, units_of
__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
__all__ = ["CaseInputs", "Loop", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
REFERENCE_USD_PER_CANDIDATE = 0.022 # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes
REFERENCE_MODEL = "deepseek-flash" # verify_batched.py's default (DEFAULT_DETECTOR_MODEL) in the batched check
@@ -76,7 +84,12 @@ CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content
ATESPACE = "default"
CASE_PATH = "/workspace/case"
INPUTS_PATH = "/workspace/inputs"
TEMPLATES = ("repo-facts", "verify", "agree")
FIXED_PATH = "/workspace/fixed"
PROJECT_PATH = "/workspace/project"
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop")
FIXED_RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, new side: the fix's lines at the fixed pin"
LOOP_MANIFEST = "benchmarks/manifests/java-spring-boot-vulnerable.toml" # the equality baseline's `improve` manifest
LOOP_CATALOG = "benchmarks/pairs/catalog.toml"
Candidate = tuple[str, str, int]
@@ -90,23 +103,40 @@ class CaseInputs:
recorded_usd: float
triage_usd: float = 0.0
ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None
fixed_ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None
def fix_ranges(repo: Path, vulnerable: str, fixed: str) -> dict[str, list[tuple[int, int]]]:
@dataclass(frozen=True)
class Loop:
"""The improvement loop appended to a Run (``--loop``, design section 6): ``repo`` at ``commit`` is this
repository at the commit under test, where ``manifest`` and ``catalog`` (paths inside it) gate the proposals;
rows of ``populations`` never propose (the train-on-test guard)."""
repo: str
commit: str
manifest: str = LOOP_MANIFEST
catalog: str = LOOP_CATALOG
populations: tuple[str, ...] = ()
def fix_ranges(repo: Path, vulnerable: str, fixed: str, side: str = "old") -> dict[str, list[tuple[int, int]]]:
"""Changed line ranges per file on the vulnerable side of the fix: `evaluate.hunks(case, "old")`, line for
line (including its attribution of a hunk under ``--- /dev/null`` to the file before it)."""
line (including its attribution of a hunk under ``--- /dev/null`` to the file before it). ``side="new"`` is
the same on the fixed side (``+++ b/`` paths, the ``+`` spans): where a fixed-pin alert sits inside the fix."""
if not (repo / ".git").exists() and not (repo / "HEAD").is_file():
raise ValueError(f"no repository at {repo}: the fix ranges cannot be computed")
diff = subprocess.run(["git", "-C", str(repo), "diff", "-U0", vulnerable, fixed], capture_output=True, text=True, check=True).stdout
ranges: dict[str, list[tuple[int, int]]] = {}
current = ""
header, group = ("--- a/", 1) if side == "old" else ("+++ b/", 3)
for line in diff.splitlines():
if line.startswith("--- a/"):
if line.startswith(header):
current = line[6:]
elif line.startswith("@@") and current:
match = re.search(r"-(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?", line)
if match:
first, length = int(match.group(1)), int(match.group(2)) if match.group(2) is not None else 1
span = match.group(group + 1)
first, length = int(match.group(group)), int(span) if span is not None else 1
ranges.setdefault(current, []).append((first, first + max(length, 1) - 1))
if not ranges:
raise ValueError(f"{repo}: the diff {vulnerable[:12]}..{fixed[:12]} changed no line on the vulnerable side")
@@ -133,10 +163,11 @@ def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
def read_cases(
validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None
validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None, *, fixed_side: bool = False
) -> list[CaseInputs]:
"""The set's cases in id order, each with its candidates, the triage-kept subset, the recorded cost and its
triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges."""
triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges (and, with
``fixed_side``, its new-side ranges, which the loop's `alerts` task marks fixed-pin alerts by)."""
subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8"))
records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]}
cases: list[CaseInputs] = []
@@ -166,6 +197,11 @@ def read_cases(
recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6),
triage_usd=_triage_usd(rows),
ranges=None if repos is None else fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"]),
fixed_ranges=(
fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"], side="new")
if repos is not None and fixed_side
else None
),
)
)
return cases
@@ -188,6 +224,9 @@ def _case_record(item: CaseInputs) -> dict[str, Any]:
if item.ranges is not None:
record["ranges"] = {path: [list(span) for span in spans] for path, spans in item.ranges.items()}
record["ranges_basis"] = RANGES_BASIS
if item.fixed_ranges is not None:
record["fixed_ranges"] = {path: [list(span) for span in spans] for path, spans in item.fixed_ranges.items()}
record["fixed_ranges_basis"] = FIXED_RANGES_BASIS
record["sites_in_set"] = [s for s in sites if s in in_set]
record["cost"] = {
"candidates_before_triage": len(item.candidates), "recorded_usd": item.recorded_usd,
@@ -213,11 +252,13 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
return pinned, inputs
def _task(template: Task, name: str, env: Mapping[str, str], bindings: Sequence[tuple[str, str]]) -> dict[str, Any]:
def _task(
template: Task, name: str, env: Mapping[str, str], bindings: Sequence[tuple[str, str]], command: Sequence[str] = ()
) -> dict[str, Any]:
spec: dict[str, Any] = {}
if template.image:
spec["image"] = template.image
spec["command"] = list(template.command)
spec["command"] = [*template.command, *command]
spec["env"] = [{"name": e.name, "value": e.value} for e in template.env] + [{"name": k, "value": v} for k, v in env.items()]
if template.resources:
kinds = {kind: values for kind, values in vars(template.resources).items() if values is not None}
@@ -257,11 +298,91 @@ def _tiebreak_entries(item: CaseInputs) -> list[dict[str, Any]]:
] # fmt: skip
def _free() -> dict[str, int]:
return {"usd": 0, "calls": 0} # model-free: budget.py refuses any call, so a stray one fails loudly
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]:
"""(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the
repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) as input."""
case_id = item.case["id"]
env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"}
task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
inputs = {"facts": f"{case_id}-facts/facts.json", "facts_summary": f"{case_id}-facts/summary.json"}
for label in ("a", "b", "c"):
inputs.update({f"pass_{label}": f"{case_id}-v{label}/units.jsonl", f"summary_{label}": f"{case_id}-v{label}/summary.json"})
inputs["agreed"] = f"{case_id}-final/agreed.json"
if alerts:
inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl"
entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()}
return task, entry
def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict, dict]:
"""(fixed-pin Workspace, Task, Run entry) of the case's `alerts`: quick-mode rules on both pins."""
case = item.case
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, ATESPACE)
fixed["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['fixed']}"}
bindings = [(_ax_name(case["id"], "vulnerable"), CASE_PATH), (fixed["metadata"]["name"], FIXED_PATH)]
bindings.append((_ax_name(case["id"], "inputs"), INPUTS_PATH))
env = {
"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo", "OUSAST_FIXED_DIR": f"{FIXED_PATH}/repo",
"OUSAST_INPUT_CASE": f"{INPUTS_PATH}/case.json", "OUSAST_VULNERABLE_PIN": str(case["vulnerable"]),
"OUSAST_FIXED_PIN": str(case["fixed"]),
} # fmt: skip
task = _task(templates["alerts"], f"alerts-{case['id']}", env, bindings)
entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl"], "budget": _free()}
entry["serialize"] = "engine"
return fixed, task, entry
def _loop(cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop) -> tuple[dict, list[dict], list[dict]]:
"""(project Workspace, Tasks, Run entries) of the loop's singletons: snapshot -> measure -> propose -> improve."""
git = [{"name": "repo", "repo": loop.repo, "dir": "repo", "depth": 1}]
project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, ATESPACE)
project["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={loop.commit}"}
bound = [(project["metadata"]["name"], PROJECT_PATH)]
env = {
"OUSAST_PROJECT_DIR": f"{PROJECT_PATH}/repo", "OUSAST_LOOP_MANIFEST": loop.manifest, "OUSAST_LOOP_CATALOG": loop.catalog,
"OUSAST_QUALIFY_POPULATIONS": json.dumps(sorted(loop.populations)),
} # fmt: skip
snapshot = _task(templates["loop"], "memory-snapshot", {}, [], ["snapshot"])
guard = {"catalog": loop.catalog, "manifest": loop.manifest, "populations": sorted(loop.populations)}
snapshot["metadata"].setdefault("annotations", {})[SNAPSHOT_ANNOTATION] = json.dumps(guard, sort_keys=True, separators=(",", ":"))
tasks = [
snapshot, _task(templates["loop"], "loop-measure", {}, [], ["measure"]),
_task(templates["loop"], "loop-propose", env, bound, ["propose"]),
_task(templates["loop"], "loop-improve", env, bound, ["improve"]),
] # fmt: skip
remembered = {f"remember_{c.case['id']}": f"{c.case['id']}-remember/memory.jsonl" for c in cases}
entries = [
{"name": "memory-snapshot", "task": "memory-snapshot", "outputs": ["rows.jsonl", "index.json"], "budget": _free()},
{"name": "loop-measure", "task": "loop-measure", "inputs": {**remembered, "snapshot": "memory-snapshot/rows.jsonl"},
"outputs": ["measure.json", "rows.jsonl"], "budget": _free()},
{"name": "loop-propose", "task": "loop-propose",
"inputs": {"measure": "loop-measure/measure.json", "rows": "loop-measure/rows.jsonl", "index": "memory-snapshot/index.json"},
"outputs": ["proposals.jsonl", "memory_proposals.jsonl", "signals.json"], "budget": _free()},
{"name": "loop-improve", "task": "loop-improve", "inputs": {"proposals": "loop-propose/proposals.jsonl"},
"outputs": ["gate.json", "journal.json"], "budget": _free(), "serialize": "engine"},
] # fmt: skip
return project, tasks, entries
def render(
cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str, *, population: str = "", split: str = ""
cases: Sequence[CaseInputs],
templates: Mapping[str, Task],
run_name: str,
command: str,
*,
population: str = "",
split: str = "",
loop: Loop | None = None,
) -> dict[str, str]:
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry."""
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each
case ends in `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the
snapshot, `loop-measure`, `loop-propose` and `loop-improve` (design section 6), all model-free."""
header = f"# generated by `{command}`; do not edit, regenerate.\n"
out: dict[str, str] = {}
tasks: list[dict[str, Any]] = []
@@ -307,6 +428,20 @@ def render(
tiebreak.extend(_tiebreak_entries(item))
tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first
entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state
for item in cases: # appended after the tie-break for the same reason
if loop is not None:
fixed, alerts_task, alerts_entry = _alerts(item, templates)
out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed])
tasks.append(alerts_task)
entries.append(alerts_entry)
remember_task, remember_entry = _remember(item, templates, population, split, loop is not None)
tasks.append(remember_task)
entries.append(remember_entry)
if loop is not None:
project, loop_tasks, loop_entries = _loop(cases, templates, loop)
out[f"workspaces/{project['metadata']['name']}.yaml"] = _dump(header, [project])
tasks.extend(loop_tasks)
entries.extend(loop_entries)
out[f"tasks/{run_name}.yaml"] = _dump(header, tasks)
annotations = {k: v for k, v in ((POPULATION_ANNOTATION, population), (SPLIT_ANNOTATION, split)) if v}
metadata: dict[str, Any] = {"name": run_name, **({"annotations": annotations} if annotations else {})}
@@ -349,14 +484,17 @@ def increment(
command: str,
repos: Path | None = None,
runner_image: Path | None = None,
loop: Loop | None = None,
) -> list[Path]:
"""Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when
``runner_image`` is given); returns the paths."""
``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths."""
if loop is not None and repos is None:
raise ValueError("--loop needs --repos: the alerts task marks fixed-pin alerts by the fix's new-side ranges")
if runner_image is not None:
repin_templates(plane, runner_image)
loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos)
files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem)
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos, fixed_side=loop is not None)
files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop)
written: list[Path] = []
for relative, text in files.items():
path = plane / relative
+20 -4
View File
@@ -20,6 +20,10 @@ Fact reuse: a facts entry is valid for (repo, pin, candidates digest, runner ima
writes as the Task annotation ``openultrasast.io/memory-key``. :func:`seed` runs before a Run: for every entry whose
Task carries a key with stored facts and no state yet, it writes ``<run>/<task>/facts.json`` and a ``done``
summary and marks the task done under the run lock (``reconciler.mark_done``), so the reconciler skips it.
The loop's ``memory-snapshot`` (a Task annotated ``openultrasast.io/memory-snapshot`` with the train-on-test guard's
parameters) is seeded the same way: a sandboxed task cannot read this store, so :func:`seed` writes the store's rows
after the guard into ``<run>/memory-snapshot/`` (:func:`.tasks.loop.write_snapshot`) and marks it done.
"""
from __future__ import annotations
@@ -44,6 +48,7 @@ log = logging.getLogger(__name__)
MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key"
POPULATION_ANNOTATION = "openultrasast.io/population"
SPLIT_ANNOTATION = "openultrasast.io/split"
SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"}
KINDS = ("facts", "verdict", "unit_cost", "alert", "proposal_outcome")
ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image")
TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split")
@@ -631,7 +636,8 @@ def ingest(run_dir: Path, store: MemoryStore | None = None) -> list[IngestResult
def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root: Path | None = None) -> list[str]:
"""Reuse stored facts for a Run's tasks whose ``openultrasast.io/memory-key`` matches (Req 6.1): each such task
without a directory or a state gets ``facts.json`` and a ``done`` summary and is marked done under the run
lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes."""
lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes.
A ``memory-snapshot`` Task gets the store's rows after the guard its annotation names, marked done likewise."""
from . import reconciler
run_spec, manifests = reconciler.load_run(Path(run_manifest))
@@ -640,10 +646,19 @@ def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root:
state = (reconciler._read_json(base / "state.json") or {}).get("tasks") or {}
seeded: list[str] = []
for entry in run_spec.tasks:
text = manifests.tasks[entry.task].metadata.annotations.get(MEMORY_KEY_ANNOTATION)
if not text or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending":
annotations = manifests.tasks[entry.task].metadata.annotations
text, snapshot = annotations.get(MEMORY_KEY_ANNOTATION), annotations.get(SNAPSHOT_ANNOTATION)
if not (text or snapshot) or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending":
continue
key = parse_memory_key(text)
if snapshot:
from .tasks.loop import write_snapshot # lazy: the guard reads the pair catalog and the benchmark manifest
info = write_snapshot(base / entry.name, store, json.loads(snapshot))
reused = {"snapshot": store.describe(), **{k: info[k] for k in ("rows", "kept", "index_digest")}}
reconciler.mark_done(run_spec.metadata.name, entry.name, reused=reused, results_root=results_root)
seeded.append(entry.name)
continue
key = parse_memory_key(str(text))
where = {"candidates_digest": key["candidates"], "image": key["image"]}
for record in sorted(store.rows(key["repo"], key["pin"], "facts", where), key=lambda r: str(r.row["run"]), reverse=True):
data = store.get_facts(str(record.row["sha256"]))
@@ -672,6 +687,7 @@ __all__ = [
"KINDS",
"MEMORY_KEY_ANNOTATION",
"POPULATION_ANNOTATION",
"SNAPSHOT_ANNOTATION",
"SPLIT_ANNOTATION",
"FileStore",
"IngestResult",
+123
View File
@@ -0,0 +1,123 @@
"""`alerts` (harnessx-removal Requirement 6.3, design section 6): what the deterministic rules say about a case.
No model, budget ``{usd: 0, calls: 0}``, one per case in a ``--loop`` Run, serialized like ``repo-facts`` (a full-tree
scan per pin). It runs the quick-mode scan -- ``preprocess_repository``, the entry-point reachability hints,
``rank_targets`` and ``quick_scan_findings`` over the image's bundled ruleset, enabled and shadow rules alike, exactly
what ``ousast scan --mode quick`` runs -- on the case's vulnerable pin and on its fixed pin, and writes one row per
(rule, file, line, pin) to ``alerts.jsonl``:
rule_id, rule_status (enabled | shadow), path, line, function, pin_role (vulnerable | fixed), pin, in_fix_range
``function`` is the enclosing declaration by the patterns ``repo-facts`` uses (so an alert joins the verify
candidates' ``path::function``; ``<global>`` outside any function); a language those patterns do not cover keeps the
scan's own reachability attribution. ``in_fix_range`` says whether the line lies inside the fix: the old-side ranges
of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fixed pin (null when ``case.json`` has
none). A fixed-pin alert inside the fix is a false alert on code the fix wrote: rule M1 of ``improve/memory.py``
counts it.
Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_WORKSPACE_DIR`` (vulnerable checkout), ``OUSAST_FIXED_DIR`` (fixed checkout),
``OUSAST_INPUT_CASE`` (``case.json``), ``OUSAST_VULNERABLE_PIN``, ``OUSAST_FIXED_PIN``. A pin whose checkout yields no
source file fails the task: an unread tree and a silent one are indistinguishable afterwards. Exit 0 done, 2 failed.
"""
from __future__ import annotations
import json
import os
import sys
import traceback
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from ...findings import quick_scan_findings
from ...mapping import analyze_entry_points, attach_reachability_hints
from ...preprocess import detect_language, preprocess_repository
from ...rank import rank_targets
from .repo_facts import DECLARATION, GLOBAL, enclosing
ROLES = ("vulnerable", "fixed")
DIRS = ("OUSAST_WORKSPACE_DIR", "OUSAST_FIXED_DIR")
PINS = ("OUSAST_VULNERABLE_PIN", "OUSAST_FIXED_PIN")
Ranges = Mapping[str, Sequence[Sequence[int]]]
def _lines(path: Path) -> list[str]:
try:
return path.read_text(encoding="utf-8", errors="replace").splitlines()
except OSError:
return []
def in_range(ranges: Ranges | None, path: str, line: int | None) -> bool | None:
if ranges is None:
return None
return line is not None and any(int(lo) <= line <= int(hi) for lo, hi in ranges.get(path, []))
def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, int]]:
"""(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line."""
_, targets = preprocess_repository(root)
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
findings = quick_scan_findings(root, targets, rank_targets(targets))
read = {"files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file())}
cache: dict[str, list[str]] = {}
rows: dict[tuple[str, str, int | None], dict[str, Any]] = {}
for finding in findings:
rule_id = finding.finding_id.split(":", 1)[0]
lines = cache.setdefault(finding.path, _lines(root / finding.path))
language = detect_language(root / finding.path)
if language in DECLARATION and finding.line and 0 < finding.line <= len(lines):
function = enclosing(lines, finding.line - 1, language)
else:
function = finding.function_name or GLOBAL
row = {
"rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line, "function": function,
"pin_role": role, "pin": pin, "in_fix_range": in_range(ranges, finding.path, finding.line),
} # fmt: skip
rows.setdefault((rule_id, finding.path, finding.line), row)
ordered = sorted(rows.values(), key=lambda r: (r["path"], r["line"] or 0, r["rule_id"]))
return ordered, read
def _summary(output_dir: Path, summary: Mapping[str, Any]) -> None:
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def main(environ: Mapping[str, str] | None = None) -> int:
env = environ if environ is not None else os.environ
output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 2, "usd": 0, "calls": 0, "usage": {}, "model": None}
try:
if output is None:
raise ValueError("OUSAST_OUTPUT_DIR is not set")
output.mkdir(parents=True, exist_ok=True)
case = json.loads(Path(env["OUSAST_INPUT_CASE"]).read_text(encoding="utf-8")) if env.get("OUSAST_INPUT_CASE") else {}
ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")}
rows: list[dict[str, Any]] = []
for role, directory, pin in zip(ROLES, DIRS, PINS, strict=True):
root = Path(env.get(directory) or "")
if not env.get(directory) or not root.is_dir():
raise ValueError(f"{directory} is not a directory: {root}")
found, read = scan(root, role, str(env.get(pin) or ""), ranges[role])
if read["files"] == 0:
raise ValueError(f"the {role} checkout {root} has no source file the scan reads: an unread tree is not a clean one")
summary.setdefault("read", {})[role] = read
summary.setdefault("alerts", {})[role] = len(found)
rows.extend(found)
summary["units_done"] += 1
(output / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
summary["status"] = "done"
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
summary["reason"] = traceback.format_exc()[-2000:]
if output is not None:
_summary(output, summary)
print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read")}), file=sys.stderr)
return 0 if summary["status"] == "done" else 2
__all__ = ["in_range", "main", "scan"]
if __name__ == "__main__":
raise SystemExit(main())
+297
View File
@@ -0,0 +1,297 @@
"""The improvement loop as plane tasks (harnessx-removal Requirement 6.3, design section 6).
``command: ["loop", <step>]``; every step is model-free with budget ``{usd: 0, calls: 0}`` (``budget.py`` refuses any
call, so a stray one fails loudly), and each writes ``summary.json`` under the task contract. Records travel as
``{"key", "version", "row"}`` lines: the store object a row came from and its version (MinIO's version id, a file's
content hash), which a proposal's provenance cites (Req 6.4).
- ``snapshot``: never runs in the sandbox, which cannot read the host's store. ``ousast plane run`` seeds it before
the Run (``memory.seed`` -> :func:`write_snapshot`): the store's rows after the train-on-test guard to
``rows.jsonl``, and ``index.json`` (store, index digest, row counts, what the guard excluded). Run in a sandbox it
fails, naming the seed that did not happen.
- ``measure``: every case's ``remember`` rows (inputs ``remember_<case>``) plus the snapshot -> ``measure.json``,
the increment's metrics for this run, the store and both (declared sites agreed, usd per candidate, dispute rates
per family, alerts per rule and pin) -- and ``rows.jsonl``, the union by row id that ``propose`` reads.
- ``propose``: ``improve/memory.py``'s rules M1/M2 over those rows after the guard (``OUSAST_QUALIFY_POPULATIONS``, the
gated manifest's cases, the pair catalog's holdout pairs), against the benchmark target's ledger and journal in
this repository at the commit under test (``OUSAST_PROJECT_DIR``) -> ``proposals.jsonl`` (edit + provenance),
``memory_proposals.jsonl`` (the sidecar) and ``signals.json`` (M3, advisory).
- ``improve``: one ``evolve.run_round`` with those proposals -- the unchanged validator and gate, the pair catalog's
holdout clause included -- on copies of the target's ledger and journal -> ``gate.json`` (the ``RoundOutcome``
and which edits came from memory), ``journal.json``, the sidecar, and ``rule_policy.json`` only when the round
was accepted; ``memory.jsonl`` holds the ``proposal_outcome`` rows the host ingests after the Run. Adopting an
accepted ledger stays a maintainer commit.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import sys
import tempfile
import traceback
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import asdict
from pathlib import Path
from typing import Any
from ..memory import REMEMBER_OUTPUT, MemoryStore, Record, validate_row
INPUT_PREFIX = "OUSAST_INPUT_REMEMBER_"
CALIBRATION = Path(".openultrasast") / "calibration" # cli.CALIBRATION_DIR: where `improve` keeps a target's ledger
ADOPTION = "adoption is a maintainer commit: copy rule_policy.json to {ledger} (README, Rule-level loop)"
# --- records -------------------------------------------------------------------------------------------------------
def _record_line(record: Record) -> str:
return json.dumps({"key": record.key, "version": record.version, "row": record.row}, sort_keys=True, separators=(",", ":")) + "\n"
def read_records(path: Path) -> list[Record]:
records: list[Record] = []
for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
if raw.strip():
item = json.loads(raw)
records.append(Record(validate_row(item["row"], f"{path}:{number}"), str(item["key"]), item.get("version")))
return records
def write_records(path: Path, records: Iterable[Record]) -> None:
path.write_text("".join(_record_line(r) for r in sorted(records, key=lambda r: str(r.row["id"]))), encoding="utf-8")
def _guard(params: Mapping[str, Any], root: Path) -> Any:
from ...benchmark import load_benchmark_manifest
from ...improve.memory import Guard, benchmark_cases
from ...pairs import load_pair_catalog, select_split
manifest = load_benchmark_manifest(root / str(params["manifest"]))
holdout = select_split(load_pair_catalog(Path(str(params["catalog"]))), "holdout")
return Guard.build([str(p) for p in params.get("populations") or []], benchmark_cases(manifest), holdout)
def write_snapshot(directory: Path, store: MemoryStore, params: Mapping[str, Any], root: Path | None = None) -> dict[str, Any]:
"""The store's rows after the guard ``params`` names (``catalog``, ``manifest``, ``populations``; paths relative to
``root``, the working directory) into ``directory``: ``rows.jsonl``, ``index.json`` and a ``done`` summary."""
from ...improve.memory import index_digest
here = Path.cwd()
try:
os.chdir(root or here) # the pair catalog names its files relative to the repository root
guard = _guard(params, Path.cwd())
finally:
os.chdir(here)
directory.mkdir(parents=True, exist_ok=False) # after the guard loaded: a failed load leaves no half-seeded task
records = store.rows()
kept, excluded = guard.apply(records)
write_records(directory / "rows.jsonl", kept)
info = {
"store": store.describe(), "index_entries": len(store.index()), "index_digest": index_digest(store), "rows": len(records),
"kept": len(kept), "excluded": excluded, "guard": dict(params),
} # fmt: skip
(directory / "index.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n", encoding="utf-8")
summary = {"status": "done", "units_done": 1, "units_total": 1, "usd": None, "calls": 0, "usage": {}, "model": None, "reused": info}
(directory / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return info
# --- measure -------------------------------------------------------------------------------------------------------
def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
"""The increment's metrics over verdict, unit-cost and alert rows (a disputed final is neither agreed nor not)."""
verdicts = [r for r in rows if r["kind"] == "verdict"]
finals = {f: sum(1 for v in verdicts if v.get("final") == f) for f in ("agreed", "rejected", "disputed")}
costs = [r.get("usd") for r in rows if r["kind"] == "unit_cost"]
usd = None if any(c is None for c in costs) else round(sum(float(c or 0) for c in costs), 6)
families: dict[str, list[int]] = {}
for v in verdicts:
tally = families.setdefault(str(v.get("family")), [0, 0])
tally[0] += 1
tally[1] += v.get("final") == "disputed"
alerts: dict[str, dict[str, int]] = {}
for a in (r for r in rows if r["kind"] == "alert"):
role = "fixed" if a.get("pin_role") == "fixed" else "vulnerable"
tally_a = alerts.setdefault(str(a.get("rule_id")), {"vulnerable": 0, "fixed": 0, "fixed_in_fix_range": 0})
tally_a[role] += 1
tally_a["fixed_in_fix_range"] += role == "fixed" and a.get("in_fix_range") is True
return {
"cases": len({(v["repo"], v["pin"]) for v in verdicts}),
"candidates": len(verdicts),
**finals,
"tiebreaks": sum(1 for v in verdicts if v.get("tiebreak")),
"declared_sites_matched": sum(1 for v in verdicts if v.get("site_match") is True),
"declared_sites_agreed": sum(1 for v in verdicts if v.get("site_match") is True and v.get("final") == "agreed"),
"usd": usd,
"usd_per_candidate": None if usd is None or not verdicts else round(usd / len(verdicts), 6),
"dispute_rate": {f: {"candidates": n, "disputed": d, "rate": round(d / n, 4)} for f, (n, d) in sorted(families.items())},
"alerts": dict(sorted(alerts.items())),
}
def measure(env: Mapping[str, str], output: Path) -> dict[str, Any]:
run = env.get("OUSAST_RUN") or "unknown"
current: list[Record] = []
read: dict[str, int] = {}
for name in sorted(k for k in env if k.startswith(INPUT_PREFIX)):
path = Path(env[name])
data = path.read_bytes()
read[name.removeprefix(INPUT_PREFIX).lower()] = len(data)
version = hashlib.sha256(data).hexdigest()
for number, raw in enumerate(data.decode("utf-8").splitlines(), 1):
if raw.strip():
row = validate_row(json.loads(raw), f"{path}:{number}")
current.append(Record(row, f"runs/{run}/{row['task']}/{REMEMBER_OUTPUT}", version))
if not current:
raise ValueError(f"read no remember rows from {len(read)} inputs ({read}): an unread run is not an empty one")
snapshot = read_records(Path(env["OUSAST_INPUT_SNAPSHOT"])) if env.get("OUSAST_INPUT_SNAPSHOT") else []
merged = {str(r.row["id"]): r for r in snapshot}
merged.update({str(r.row["id"]): r for r in current}) # this run's row wins over its earlier ingest
write_records(output / "rows.jsonl", merged.values())
report = {
"run": run, "inputs_bytes": read, "rows": {"run": len(current), "snapshot": len(snapshot), "union": len(merged)},
"run_metrics": metrics([r.row for r in current]), "store_metrics": metrics([r.row for r in snapshot]),
"union_metrics": metrics([r.row for r in merged.values()]),
} # fmt: skip
(output / "measure.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return {"rows": report["rows"]}
# --- propose and improve -------------------------------------------------------------------------------------------
def _project(env: Mapping[str, str]) -> tuple[Any, Path, Path, Path]:
"""(manifest, target, ledger, journal) in the repository at the commit under test; the working directory
becomes the repository root, where the pair catalog's relative paths resolve."""
from ...benchmark import load_benchmark_manifest, resolve_benchmark_source
project = Path(env.get("OUSAST_PROJECT_DIR") or "")
if not env.get("OUSAST_PROJECT_DIR") or not (project / str(env.get("OUSAST_LOOP_MANIFEST"))).is_file():
raise ValueError(f"OUSAST_PROJECT_DIR {project} holds no benchmark manifest {env.get('OUSAST_LOOP_MANIFEST')}")
os.chdir(project)
manifest_path = Path(str(env["OUSAST_LOOP_MANIFEST"]))
manifest = load_benchmark_manifest(manifest_path)
target = resolve_benchmark_source(manifest_path, manifest)
ledger = target / CALIBRATION / "rule_policy.json"
return manifest, target, ledger, ledger.with_name("improve_journal.json")
def _params(env: Mapping[str, str]) -> dict[str, Any]:
populations = json.loads(env.get("OUSAST_QUALIFY_POPULATIONS") or "[]")
return {"catalog": env.get("OUSAST_LOOP_CATALOG"), "manifest": env.get("OUSAST_LOOP_MANIFEST"), "populations": populations}
def propose(env: Mapping[str, str], output: Path) -> dict[str, Any]:
from ...improve.journal import load_journal, next_round_index
from ...improve.memory import disputed_families, propose_from_memory, write_sidecar, write_signals
from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset, read_rule_ledger
records = read_records(Path(env["OUSAST_INPUT_ROWS"]))
index = json.loads(Path(env["OUSAST_INPUT_INDEX"]).read_text(encoding="utf-8"))
_, _, ledger, journal = _project(env)
guard = _guard(_params(env), Path.cwd())
rules = load_ruleset(DEFAULT_RULESET_DIR, ledger if ledger.is_file() else None)
rounds = load_journal(journal)
found = propose_from_memory(
records, {r.rule_id: r for r in rules}, read_rule_ledger(ledger), rounds, excluded=guard, index_digest=str(index["index_digest"])
)
lines = [json.dumps({"edit": asdict(p.edit), "provenance": p.provenance}, sort_keys=True) + "\n" for p in found]
(output / "proposals.jsonl").write_text("".join(lines), encoding="utf-8")
sidecar = output / "journal.json"
write_sidecar(sidecar, next_round_index(rounds), found)
sidecar.with_name("memory_proposals.jsonl").touch() # present when nothing was proposed, as a declared output
write_signals(output, disputed_families(records, guard))
kept, excluded = guard.apply(records)
return {"proposals": len(found), "rows": len(records), "kept": len(kept), "excluded": excluded["rows"]}
def improve(env: Mapping[str, str], output: Path) -> dict[str, Any]:
from ...gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
from ...improve.evolve import run_round
from ...improve.memory import MemoryProposal, outcome_of, outcome_rows, proposal_id_of
from ...improve.validator import RuleStatusEdit
from ...pairs import load_pair_catalog, select_split
from ...policy import load_policy
from ...ruleset import DEFAULT_RULESET_DIR
path = Path(env["OUSAST_INPUT_PROPOSALS"])
items = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
proposals = [MemoryProposal(RuleStatusEdit(**item["edit"]), item["provenance"]) for item in items]
manifest, target, ledger, journal = _project(env)
pairs = select_split(load_pair_catalog(Path(str(env["OUSAST_LOOP_CATALOG"]))), "holdout")
with tempfile.TemporaryDirectory(prefix="ousast-loop-") as scratch:
work_ledger = Path(scratch) / "rule_policy.json"
if ledger.is_file():
shutil.copyfile(ledger, work_ledger)
if journal.is_file():
shutil.copyfile(journal, output / "journal.json")
outcome = run_round(
target, manifest, ledger_path=work_ledger, journal_path=output / "journal.json", ruleset_dir=DEFAULT_RULESET_DIR,
policy=load_policy(), recall_floor=RECALL_FLOOR, fp_ceiling=FALSE_POSITIVE_CEILING, pair_cases=pairs, proposals=proposals,
) # fmt: skip
if outcome.accepted and work_ledger.is_file():
shutil.copyfile(work_ledger, output / "rule_policy.json")
(output / "journal.json").touch()
(output / "memory_proposals.jsonl").touch()
memory_edits = [e.key() for e in outcome.edits if proposal_id_of(e)]
verdict = {
"accepted": outcome.accepted, "reason": outcome.reason, "round": outcome.round, "proposals": len(proposals),
"memory_edits": memory_edits, "benchmark_edits": [e.key() for e in outcome.edits if not proposal_id_of(e)],
"manifest": env.get("OUSAST_LOOP_MANIFEST"), "outcome": asdict(outcome),
"adoption": ADOPTION.format(ledger=ledger.as_posix()) if outcome.accepted else None,
} # fmt: skip
(output / "gate.json").write_text(json.dumps(verdict, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8")
status = outcome_of(outcome.reason, outcome.accepted)
by_id = {p.proposal_id: p for p in proposals}
gate_run = f"plane:{env.get('OUSAST_RUN') or 'unknown'}"
rows = outcome_rows(outcome.edits, status, outcome.round, outcome.reason, by_id, gate_run) if status else []
(output / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
return {"accepted": outcome.accepted, "reason": outcome.reason, "edits": len(outcome.edits), "memory_edits": len(memory_edits)}
def snapshot(env: Mapping[str, str], output: Path) -> dict[str, Any]:
raise RuntimeError(
"memory-snapshot ran in the sandbox: `ousast plane run` writes it on the host from the memory store before the Run "
"starts (memory.seed), so the seed did not happen; rerun through `ousast plane run`"
)
STEPS: dict[str, Callable[[Mapping[str, str], Path], dict[str, Any]]] = {
"snapshot": snapshot, "measure": measure, "propose": propose, "improve": improve,
} # fmt: skip
def main(argv: Sequence[str] | None = None, environ: Mapping[str, str] | None = None) -> int:
args = list(sys.argv[1:] if argv is None else argv)
env = environ if environ is not None else os.environ
output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None}
here = Path.cwd()
try:
if output is None:
raise ValueError("OUSAST_OUTPUT_DIR is not set")
if len(args) != 1 or args[0] not in STEPS:
raise ValueError(f"expected one step of {sorted(STEPS)}, got {args}")
output.mkdir(parents=True, exist_ok=True)
summary.update(step=args[0], **STEPS[args[0]](env, output))
summary.update(status="done", units_done=1)
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
summary["reason"] = traceback.format_exc()[-2000:]
finally:
os.chdir(here)
if output is not None:
(output / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8")
shown = {k: summary.get(k) for k in ("status", "step", "proposals", "accepted", "reason", "rows")}
print(json.dumps(shown, default=str), file=sys.stderr)
return 0 if summary["status"] == "done" else 2
__all__ = ["main", "measure", "metrics", "propose", "improve", "read_records", "write_records", "write_snapshot"]
if __name__ == "__main__":
raise SystemExit(main())
+30 -4
View File
@@ -9,7 +9,8 @@ artifacts, so the reconciler and the runner stay content-blind (ai-service-plane
Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`,
`population`, `split` and `image` (the runner image digest the case's tasks ran on).
Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS`,
Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS` (and `OUSAST_INPUT_FACTS_SUMMARY`,
whose reused or recorded `image` the facts row carries; without one named, the row is omitted),
`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes'
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound
Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from
@@ -40,6 +41,7 @@ from .repo_facts import _digest
PASSES = ("a", "b", "c")
UNKNOWN = "unknown"
ALERT_FIELDS = ("rule_id", "rule_status", "path", "line", "function", "pin_role")
@dataclass(frozen=True)
@@ -108,8 +110,13 @@ def rows_for(
} # fmt: skip
rows.append(ctx.row("unit_cost", f"{label}:{unit.get('path')}", **fields))
for alert in alerts:
fields = {k: alert.get(k) for k in ("rule_id", "rule_status", "path", "line", "function", "pin_role")}
rows.append(ctx.row("alert", f"{fields['rule_id']}:{fields['path']}:{fields['line']}", **fields))
fields = {k: alert.get(k) for k in ALERT_FIELDS}
if "in_fix_range" in alert: # the loop's `alerts` task marks it; improve/memory.py's M1 reads it
fields["in_fix_range"] = alert["in_fix_range"]
if alert.get("pin"): # a fixed-pin alert is a row of the pin it was raised on, not of the case's vulnerable pin
fields["pin"] = str(alert["pin"])
subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "")
rows.append(ctx.row("alert", subject, **fields))
return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows]
@@ -202,6 +209,19 @@ def _path(env: Mapping[str, str], name: str) -> Path | None:
return Path(env[name]) if env.get(name) else None
def _facts_image(env: Mapping[str, str], own: str) -> str | None:
"""The image the case's facts were computed on: without `OUSAST_INPUT_FACTS_SUMMARY` this task's own; with it,
the reused entry's image or the image the facts task recorded, else None (unknown)."""
path = _path(env, "OUSAST_INPUT_FACTS_SUMMARY")
if path is None:
return own
summary = _json(path) or {}
reused = summary.get("reused")
reused = reused if isinstance(reused, dict) else {}
image = reused.get("image") or summary.get("image")
return image_digest(str(image)) if image else None
def _units(path: Path | None) -> list[dict[str, Any]]:
return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path)
@@ -226,7 +246,13 @@ def main(environ: Mapping[str, str] | None = None) -> int:
passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES}
models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES}
agreed = _json(_path(env, "OUSAST_INPUT_AGREED"))
rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")))
rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")))
facts_image = _facts_image(env, ctx.image)
if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on
facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split)
rows = rows_for(facts_ctx, facts=facts, passes={}, models={}, agreed=None) + rows
elif facts is not None:
summary["facts_row"] = "omitted: the facts task's summary names no image, and a reuse key must not guess it"
(output_dir / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
if facts is not None:
(output_dir / "facts.json").write_bytes(facts)
+10 -4
View File
@@ -23,6 +23,8 @@ from collections.abc import Iterable, Mapping, Sequence
from pathlib import Path
from typing import Any
import yaml
from ...model.layout import is_test_path, is_vendored, layout_facts
from ...preprocess import IGNORED_DIRS, detect_language
@@ -259,12 +261,13 @@ def assemble_facts(
}
def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | None = None) -> dict[str, Any]:
def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | None = None, image: str | None = None) -> dict[str, Any]:
"""Compute the facts for ``workspace`` into ``output_dir``; returns the written summary.
``candidates`` rows are ``{"path", "function"}`` (workspace-relative POSIX path). ``None`` means every
function every product file defines. A `units.jsonl` row from an earlier run over the same candidate set
is reused, not recomputed.
is reused, not recomputed. ``image`` (the runner image, from `AX_TASK_YAML`) goes into the summary: the
`remember` task keys the facts row by the image the facts were computed on.
"""
output_dir.mkdir(parents=True, exist_ok=True)
units_path = output_dir / "units.jsonl"
@@ -307,7 +310,8 @@ def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | No
raise TaskFailed(str(exc)) from exc
facts = assemble_facts(rows, wanted)
_write_json(output_dir / "facts.json", facts)
return write_summary(output_dir, status="done", units_done=len(rows), units_total=len(files))
extra = {"image": image} if image else {}
return write_summary(output_dir, status="done", units_done=len(rows), units_total=len(files), **extra)
def _load_candidates(path: str | None) -> list[dict[str, Any]] | None:
@@ -332,7 +336,9 @@ def main(environ: Mapping[str, str] | None = None) -> int:
workspace = Path(env.get("OUSAST_WORKSPACE_DIR") or "")
if not env.get("OUSAST_WORKSPACE_DIR") or not workspace.is_dir():
raise ValueError(f"OUSAST_WORKSPACE_DIR is not a directory: {workspace}")
summary = run(workspace, output_dir, _load_candidates(env.get("OUSAST_INPUT_CANDIDATES")))
task = yaml.safe_load(env.get("AX_TASK_YAML") or "{}")
image = str((task.get("spec") or {}).get("image") or "") if isinstance(task, dict) else ""
summary = run(workspace, output_dir, _load_candidates(env.get("OUSAST_INPUT_CANDIDATES")), image or None)
except TaskFailed as exc:
print(f"repo-facts failed: {exc}", file=sys.stderr)
return 2