mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
feat(plane): the improvement loop as a plane Run (harnessx-removal task 4)
`ousast plane workspaces --validation-set ... --loop` appends the loop to a Run:
per case `alerts` (plane/tasks/alerts.py: the quick-mode scan on the vulnerable
and the fixed pin, a fixed-pin Workspace per case, alerts attributed to their
enclosing function by the repo-facts patterns, `in_fix_range` from the fix's
old-side and new-side ranges, failing on an unread tree) feeding `remember`,
then `memory-snapshot` -> `loop-measure` -> `loop-propose` -> `loop-improve`
(plane/tasks/loop.py). Every new task binds no Model and has budget
{usd: 0, calls: 0}. The snapshot is the store's rows after the train-on-test
guard, written on the host by memory.seed (a sandboxed run of it fails loudly);
propose runs M1/M2 against this repository at the commit under test; improve
runs one unchanged evolve.run_round and delivers gate.json, journal.json, the
sidecar, rule_policy.json only when accepted, and proposal_outcome rows that
the host ingests. Adoption stays a maintainer commit.
`remember` is now a Run entry per case (the image contains it), with the facts
row keyed by the image the facts were computed on (repo-facts records it in
its summary; unknown -> the row is omitted, never guessed). Alert rows carry
their own pin; improve/memory.py counts a repeated alert once.
Runner image rebuilt (+0.39 MB: only the new modules; the quick scan needs no
extra and the rulesets were already in the package), pushed, plane/ re-pinned
and regenerated (validation-46: 105 entries). Fixture test runs the whole loop
graph through the fake ax, whose new `exec` mode runs the real task modules.
Live: one loop Run over three validation-46 cases on the local ax, recorded in
benchmarks/measurements/2026-09-30-harnessx-removal-loop/ (all done, $0 new
spend, 0 proposals, gate verdict no_proposals).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d104166697
commit
b68fe27c74
@@ -0,0 +1,123 @@
|
||||
"""`alerts` (harnessx-removal Requirement 6.3, design section 6): what the deterministic rules say about a case.
|
||||
|
||||
No model, budget ``{usd: 0, calls: 0}``, one per case in a ``--loop`` Run, serialized like ``repo-facts`` (a full-tree
|
||||
scan per pin). It runs the quick-mode scan -- ``preprocess_repository``, the entry-point reachability hints,
|
||||
``rank_targets`` and ``quick_scan_findings`` over the image's bundled ruleset, enabled and shadow rules alike, exactly
|
||||
what ``ousast scan --mode quick`` runs -- on the case's vulnerable pin and on its fixed pin, and writes one row per
|
||||
(rule, file, line, pin) to ``alerts.jsonl``:
|
||||
|
||||
rule_id, rule_status (enabled | shadow), path, line, function, pin_role (vulnerable | fixed), pin, in_fix_range
|
||||
|
||||
``function`` is the enclosing declaration by the patterns ``repo-facts`` uses (so an alert joins the verify
|
||||
candidates' ``path::function``; ``<global>`` outside any function); a language those patterns do not cover keeps the
|
||||
scan's own reachability attribution. ``in_fix_range`` says whether the line lies inside the fix: the old-side ranges
|
||||
of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fixed pin (null when ``case.json`` has
|
||||
none). A fixed-pin alert inside the fix is a false alert on code the fix wrote: rule M1 of ``improve/memory.py``
|
||||
counts it.
|
||||
|
||||
Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_WORKSPACE_DIR`` (vulnerable checkout), ``OUSAST_FIXED_DIR`` (fixed checkout),
|
||||
``OUSAST_INPUT_CASE`` (``case.json``), ``OUSAST_VULNERABLE_PIN``, ``OUSAST_FIXED_PIN``. A pin whose checkout yields no
|
||||
source file fails the task: an unread tree and a silent one are indistinguishable afterwards. Exit 0 done, 2 failed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ...findings import quick_scan_findings
|
||||
from ...mapping import analyze_entry_points, attach_reachability_hints
|
||||
from ...preprocess import detect_language, preprocess_repository
|
||||
from ...rank import rank_targets
|
||||
from .repo_facts import DECLARATION, GLOBAL, enclosing
|
||||
|
||||
ROLES = ("vulnerable", "fixed")
|
||||
DIRS = ("OUSAST_WORKSPACE_DIR", "OUSAST_FIXED_DIR")
|
||||
PINS = ("OUSAST_VULNERABLE_PIN", "OUSAST_FIXED_PIN")
|
||||
Ranges = Mapping[str, Sequence[Sequence[int]]]
|
||||
|
||||
|
||||
def _lines(path: Path) -> list[str]:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def in_range(ranges: Ranges | None, path: str, line: int | None) -> bool | None:
|
||||
if ranges is None:
|
||||
return None
|
||||
return line is not None and any(int(lo) <= line <= int(hi) for lo, hi in ranges.get(path, []))
|
||||
|
||||
|
||||
def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
||||
"""(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line."""
|
||||
_, targets = preprocess_repository(root)
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
read = {"files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file())}
|
||||
cache: dict[str, list[str]] = {}
|
||||
rows: dict[tuple[str, str, int | None], dict[str, Any]] = {}
|
||||
for finding in findings:
|
||||
rule_id = finding.finding_id.split(":", 1)[0]
|
||||
lines = cache.setdefault(finding.path, _lines(root / finding.path))
|
||||
language = detect_language(root / finding.path)
|
||||
if language in DECLARATION and finding.line and 0 < finding.line <= len(lines):
|
||||
function = enclosing(lines, finding.line - 1, language)
|
||||
else:
|
||||
function = finding.function_name or GLOBAL
|
||||
row = {
|
||||
"rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line, "function": function,
|
||||
"pin_role": role, "pin": pin, "in_fix_range": in_range(ranges, finding.path, finding.line),
|
||||
} # fmt: skip
|
||||
rows.setdefault((rule_id, finding.path, finding.line), row)
|
||||
ordered = sorted(rows.values(), key=lambda r: (r["path"], r["line"] or 0, r["rule_id"]))
|
||||
return ordered, read
|
||||
|
||||
|
||||
def _summary(output_dir: Path, summary: Mapping[str, Any]) -> None:
|
||||
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
env = environ if environ is not None else os.environ
|
||||
output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None
|
||||
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 2, "usd": 0, "calls": 0, "usage": {}, "model": None}
|
||||
try:
|
||||
if output is None:
|
||||
raise ValueError("OUSAST_OUTPUT_DIR is not set")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
case = json.loads(Path(env["OUSAST_INPUT_CASE"]).read_text(encoding="utf-8")) if env.get("OUSAST_INPUT_CASE") else {}
|
||||
ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")}
|
||||
rows: list[dict[str, Any]] = []
|
||||
for role, directory, pin in zip(ROLES, DIRS, PINS, strict=True):
|
||||
root = Path(env.get(directory) or "")
|
||||
if not env.get(directory) or not root.is_dir():
|
||||
raise ValueError(f"{directory} is not a directory: {root}")
|
||||
found, read = scan(root, role, str(env.get(pin) or ""), ranges[role])
|
||||
if read["files"] == 0:
|
||||
raise ValueError(f"the {role} checkout {root} has no source file the scan reads: an unread tree is not a clean one")
|
||||
summary.setdefault("read", {})[role] = read
|
||||
summary.setdefault("alerts", {})[role] = len(found)
|
||||
rows.extend(found)
|
||||
summary["units_done"] += 1
|
||||
(output / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
|
||||
summary["status"] = "done"
|
||||
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
|
||||
summary["reason"] = traceback.format_exc()[-2000:]
|
||||
if output is not None:
|
||||
_summary(output, summary)
|
||||
print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read")}), file=sys.stderr)
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["in_range", "main", "scan"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,297 @@
|
||||
"""The improvement loop as plane tasks (harnessx-removal Requirement 6.3, design section 6).
|
||||
|
||||
``command: ["loop", <step>]``; every step is model-free with budget ``{usd: 0, calls: 0}`` (``budget.py`` refuses any
|
||||
call, so a stray one fails loudly), and each writes ``summary.json`` under the task contract. Records travel as
|
||||
``{"key", "version", "row"}`` lines: the store object a row came from and its version (MinIO's version id, a file's
|
||||
content hash), which a proposal's provenance cites (Req 6.4).
|
||||
|
||||
- ``snapshot``: never runs in the sandbox, which cannot read the host's store. ``ousast plane run`` seeds it before
|
||||
the Run (``memory.seed`` -> :func:`write_snapshot`): the store's rows after the train-on-test guard to
|
||||
``rows.jsonl``, and ``index.json`` (store, index digest, row counts, what the guard excluded). Run in a sandbox it
|
||||
fails, naming the seed that did not happen.
|
||||
- ``measure``: every case's ``remember`` rows (inputs ``remember_<case>``) plus the snapshot -> ``measure.json``,
|
||||
the increment's metrics for this run, the store and both (declared sites agreed, usd per candidate, dispute rates
|
||||
per family, alerts per rule and pin) -- and ``rows.jsonl``, the union by row id that ``propose`` reads.
|
||||
- ``propose``: ``improve/memory.py``'s rules M1/M2 over those rows after the guard (``OUSAST_QUALIFY_POPULATIONS``, the
|
||||
gated manifest's cases, the pair catalog's holdout pairs), against the benchmark target's ledger and journal in
|
||||
this repository at the commit under test (``OUSAST_PROJECT_DIR``) -> ``proposals.jsonl`` (edit + provenance),
|
||||
``memory_proposals.jsonl`` (the sidecar) and ``signals.json`` (M3, advisory).
|
||||
- ``improve``: one ``evolve.run_round`` with those proposals -- the unchanged validator and gate, the pair catalog's
|
||||
holdout clause included -- on copies of the target's ledger and journal -> ``gate.json`` (the ``RoundOutcome``
|
||||
and which edits came from memory), ``journal.json``, the sidecar, and ``rule_policy.json`` only when the round
|
||||
was accepted; ``memory.jsonl`` holds the ``proposal_outcome`` rows the host ingests after the Run. Adopting an
|
||||
accepted ledger stays a maintainer commit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..memory import REMEMBER_OUTPUT, MemoryStore, Record, validate_row
|
||||
|
||||
INPUT_PREFIX = "OUSAST_INPUT_REMEMBER_"
|
||||
CALIBRATION = Path(".openultrasast") / "calibration" # cli.CALIBRATION_DIR: where `improve` keeps a target's ledger
|
||||
ADOPTION = "adoption is a maintainer commit: copy rule_policy.json to {ledger} (README, Rule-level loop)"
|
||||
|
||||
|
||||
# --- records -------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _record_line(record: Record) -> str:
|
||||
return json.dumps({"key": record.key, "version": record.version, "row": record.row}, sort_keys=True, separators=(",", ":")) + "\n"
|
||||
|
||||
|
||||
def read_records(path: Path) -> list[Record]:
|
||||
records: list[Record] = []
|
||||
for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
if raw.strip():
|
||||
item = json.loads(raw)
|
||||
records.append(Record(validate_row(item["row"], f"{path}:{number}"), str(item["key"]), item.get("version")))
|
||||
return records
|
||||
|
||||
|
||||
def write_records(path: Path, records: Iterable[Record]) -> None:
|
||||
path.write_text("".join(_record_line(r) for r in sorted(records, key=lambda r: str(r.row["id"]))), encoding="utf-8")
|
||||
|
||||
|
||||
def _guard(params: Mapping[str, Any], root: Path) -> Any:
|
||||
from ...benchmark import load_benchmark_manifest
|
||||
from ...improve.memory import Guard, benchmark_cases
|
||||
from ...pairs import load_pair_catalog, select_split
|
||||
|
||||
manifest = load_benchmark_manifest(root / str(params["manifest"]))
|
||||
holdout = select_split(load_pair_catalog(Path(str(params["catalog"]))), "holdout")
|
||||
return Guard.build([str(p) for p in params.get("populations") or []], benchmark_cases(manifest), holdout)
|
||||
|
||||
|
||||
def write_snapshot(directory: Path, store: MemoryStore, params: Mapping[str, Any], root: Path | None = None) -> dict[str, Any]:
|
||||
"""The store's rows after the guard ``params`` names (``catalog``, ``manifest``, ``populations``; paths relative to
|
||||
``root``, the working directory) into ``directory``: ``rows.jsonl``, ``index.json`` and a ``done`` summary."""
|
||||
from ...improve.memory import index_digest
|
||||
|
||||
here = Path.cwd()
|
||||
try:
|
||||
os.chdir(root or here) # the pair catalog names its files relative to the repository root
|
||||
guard = _guard(params, Path.cwd())
|
||||
finally:
|
||||
os.chdir(here)
|
||||
directory.mkdir(parents=True, exist_ok=False) # after the guard loaded: a failed load leaves no half-seeded task
|
||||
records = store.rows()
|
||||
kept, excluded = guard.apply(records)
|
||||
write_records(directory / "rows.jsonl", kept)
|
||||
info = {
|
||||
"store": store.describe(), "index_entries": len(store.index()), "index_digest": index_digest(store), "rows": len(records),
|
||||
"kept": len(kept), "excluded": excluded, "guard": dict(params),
|
||||
} # fmt: skip
|
||||
(directory / "index.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
summary = {"status": "done", "units_done": 1, "units_total": 1, "usd": None, "calls": 0, "usage": {}, "model": None, "reused": info}
|
||||
(directory / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return info
|
||||
|
||||
|
||||
# --- measure -------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
"""The increment's metrics over verdict, unit-cost and alert rows (a disputed final is neither agreed nor not)."""
|
||||
verdicts = [r for r in rows if r["kind"] == "verdict"]
|
||||
finals = {f: sum(1 for v in verdicts if v.get("final") == f) for f in ("agreed", "rejected", "disputed")}
|
||||
costs = [r.get("usd") for r in rows if r["kind"] == "unit_cost"]
|
||||
usd = None if any(c is None for c in costs) else round(sum(float(c or 0) for c in costs), 6)
|
||||
families: dict[str, list[int]] = {}
|
||||
for v in verdicts:
|
||||
tally = families.setdefault(str(v.get("family")), [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += v.get("final") == "disputed"
|
||||
alerts: dict[str, dict[str, int]] = {}
|
||||
for a in (r for r in rows if r["kind"] == "alert"):
|
||||
role = "fixed" if a.get("pin_role") == "fixed" else "vulnerable"
|
||||
tally_a = alerts.setdefault(str(a.get("rule_id")), {"vulnerable": 0, "fixed": 0, "fixed_in_fix_range": 0})
|
||||
tally_a[role] += 1
|
||||
tally_a["fixed_in_fix_range"] += role == "fixed" and a.get("in_fix_range") is True
|
||||
return {
|
||||
"cases": len({(v["repo"], v["pin"]) for v in verdicts}),
|
||||
"candidates": len(verdicts),
|
||||
**finals,
|
||||
"tiebreaks": sum(1 for v in verdicts if v.get("tiebreak")),
|
||||
"declared_sites_matched": sum(1 for v in verdicts if v.get("site_match") is True),
|
||||
"declared_sites_agreed": sum(1 for v in verdicts if v.get("site_match") is True and v.get("final") == "agreed"),
|
||||
"usd": usd,
|
||||
"usd_per_candidate": None if usd is None or not verdicts else round(usd / len(verdicts), 6),
|
||||
"dispute_rate": {f: {"candidates": n, "disputed": d, "rate": round(d / n, 4)} for f, (n, d) in sorted(families.items())},
|
||||
"alerts": dict(sorted(alerts.items())),
|
||||
}
|
||||
|
||||
|
||||
def measure(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
run = env.get("OUSAST_RUN") or "unknown"
|
||||
current: list[Record] = []
|
||||
read: dict[str, int] = {}
|
||||
for name in sorted(k for k in env if k.startswith(INPUT_PREFIX)):
|
||||
path = Path(env[name])
|
||||
data = path.read_bytes()
|
||||
read[name.removeprefix(INPUT_PREFIX).lower()] = len(data)
|
||||
version = hashlib.sha256(data).hexdigest()
|
||||
for number, raw in enumerate(data.decode("utf-8").splitlines(), 1):
|
||||
if raw.strip():
|
||||
row = validate_row(json.loads(raw), f"{path}:{number}")
|
||||
current.append(Record(row, f"runs/{run}/{row['task']}/{REMEMBER_OUTPUT}", version))
|
||||
if not current:
|
||||
raise ValueError(f"read no remember rows from {len(read)} inputs ({read}): an unread run is not an empty one")
|
||||
snapshot = read_records(Path(env["OUSAST_INPUT_SNAPSHOT"])) if env.get("OUSAST_INPUT_SNAPSHOT") else []
|
||||
merged = {str(r.row["id"]): r for r in snapshot}
|
||||
merged.update({str(r.row["id"]): r for r in current}) # this run's row wins over its earlier ingest
|
||||
write_records(output / "rows.jsonl", merged.values())
|
||||
report = {
|
||||
"run": run, "inputs_bytes": read, "rows": {"run": len(current), "snapshot": len(snapshot), "union": len(merged)},
|
||||
"run_metrics": metrics([r.row for r in current]), "store_metrics": metrics([r.row for r in snapshot]),
|
||||
"union_metrics": metrics([r.row for r in merged.values()]),
|
||||
} # fmt: skip
|
||||
(output / "measure.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return {"rows": report["rows"]}
|
||||
|
||||
|
||||
# --- propose and improve -------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _project(env: Mapping[str, str]) -> tuple[Any, Path, Path, Path]:
|
||||
"""(manifest, target, ledger, journal) in the repository at the commit under test; the working directory
|
||||
becomes the repository root, where the pair catalog's relative paths resolve."""
|
||||
from ...benchmark import load_benchmark_manifest, resolve_benchmark_source
|
||||
|
||||
project = Path(env.get("OUSAST_PROJECT_DIR") or "")
|
||||
if not env.get("OUSAST_PROJECT_DIR") or not (project / str(env.get("OUSAST_LOOP_MANIFEST"))).is_file():
|
||||
raise ValueError(f"OUSAST_PROJECT_DIR {project} holds no benchmark manifest {env.get('OUSAST_LOOP_MANIFEST')}")
|
||||
os.chdir(project)
|
||||
manifest_path = Path(str(env["OUSAST_LOOP_MANIFEST"]))
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
ledger = target / CALIBRATION / "rule_policy.json"
|
||||
return manifest, target, ledger, ledger.with_name("improve_journal.json")
|
||||
|
||||
|
||||
def _params(env: Mapping[str, str]) -> dict[str, Any]:
|
||||
populations = json.loads(env.get("OUSAST_QUALIFY_POPULATIONS") or "[]")
|
||||
return {"catalog": env.get("OUSAST_LOOP_CATALOG"), "manifest": env.get("OUSAST_LOOP_MANIFEST"), "populations": populations}
|
||||
|
||||
|
||||
def propose(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
from ...improve.journal import load_journal, next_round_index
|
||||
from ...improve.memory import disputed_families, propose_from_memory, write_sidecar, write_signals
|
||||
from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset, read_rule_ledger
|
||||
|
||||
records = read_records(Path(env["OUSAST_INPUT_ROWS"]))
|
||||
index = json.loads(Path(env["OUSAST_INPUT_INDEX"]).read_text(encoding="utf-8"))
|
||||
_, _, ledger, journal = _project(env)
|
||||
guard = _guard(_params(env), Path.cwd())
|
||||
rules = load_ruleset(DEFAULT_RULESET_DIR, ledger if ledger.is_file() else None)
|
||||
rounds = load_journal(journal)
|
||||
found = propose_from_memory(
|
||||
records, {r.rule_id: r for r in rules}, read_rule_ledger(ledger), rounds, excluded=guard, index_digest=str(index["index_digest"])
|
||||
)
|
||||
lines = [json.dumps({"edit": asdict(p.edit), "provenance": p.provenance}, sort_keys=True) + "\n" for p in found]
|
||||
(output / "proposals.jsonl").write_text("".join(lines), encoding="utf-8")
|
||||
sidecar = output / "journal.json"
|
||||
write_sidecar(sidecar, next_round_index(rounds), found)
|
||||
sidecar.with_name("memory_proposals.jsonl").touch() # present when nothing was proposed, as a declared output
|
||||
write_signals(output, disputed_families(records, guard))
|
||||
kept, excluded = guard.apply(records)
|
||||
return {"proposals": len(found), "rows": len(records), "kept": len(kept), "excluded": excluded["rows"]}
|
||||
|
||||
|
||||
def improve(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
from ...gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
from ...improve.evolve import run_round
|
||||
from ...improve.memory import MemoryProposal, outcome_of, outcome_rows, proposal_id_of
|
||||
from ...improve.validator import RuleStatusEdit
|
||||
from ...pairs import load_pair_catalog, select_split
|
||||
from ...policy import load_policy
|
||||
from ...ruleset import DEFAULT_RULESET_DIR
|
||||
|
||||
path = Path(env["OUSAST_INPUT_PROPOSALS"])
|
||||
items = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
proposals = [MemoryProposal(RuleStatusEdit(**item["edit"]), item["provenance"]) for item in items]
|
||||
manifest, target, ledger, journal = _project(env)
|
||||
pairs = select_split(load_pair_catalog(Path(str(env["OUSAST_LOOP_CATALOG"]))), "holdout")
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-loop-") as scratch:
|
||||
work_ledger = Path(scratch) / "rule_policy.json"
|
||||
if ledger.is_file():
|
||||
shutil.copyfile(ledger, work_ledger)
|
||||
if journal.is_file():
|
||||
shutil.copyfile(journal, output / "journal.json")
|
||||
outcome = run_round(
|
||||
target, manifest, ledger_path=work_ledger, journal_path=output / "journal.json", ruleset_dir=DEFAULT_RULESET_DIR,
|
||||
policy=load_policy(), recall_floor=RECALL_FLOOR, fp_ceiling=FALSE_POSITIVE_CEILING, pair_cases=pairs, proposals=proposals,
|
||||
) # fmt: skip
|
||||
if outcome.accepted and work_ledger.is_file():
|
||||
shutil.copyfile(work_ledger, output / "rule_policy.json")
|
||||
(output / "journal.json").touch()
|
||||
(output / "memory_proposals.jsonl").touch()
|
||||
memory_edits = [e.key() for e in outcome.edits if proposal_id_of(e)]
|
||||
verdict = {
|
||||
"accepted": outcome.accepted, "reason": outcome.reason, "round": outcome.round, "proposals": len(proposals),
|
||||
"memory_edits": memory_edits, "benchmark_edits": [e.key() for e in outcome.edits if not proposal_id_of(e)],
|
||||
"manifest": env.get("OUSAST_LOOP_MANIFEST"), "outcome": asdict(outcome),
|
||||
"adoption": ADOPTION.format(ledger=ledger.as_posix()) if outcome.accepted else None,
|
||||
} # fmt: skip
|
||||
(output / "gate.json").write_text(json.dumps(verdict, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8")
|
||||
status = outcome_of(outcome.reason, outcome.accepted)
|
||||
by_id = {p.proposal_id: p for p in proposals}
|
||||
gate_run = f"plane:{env.get('OUSAST_RUN') or 'unknown'}"
|
||||
rows = outcome_rows(outcome.edits, status, outcome.round, outcome.reason, by_id, gate_run) if status else []
|
||||
(output / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
|
||||
return {"accepted": outcome.accepted, "reason": outcome.reason, "edits": len(outcome.edits), "memory_edits": len(memory_edits)}
|
||||
|
||||
|
||||
def snapshot(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
raise RuntimeError(
|
||||
"memory-snapshot ran in the sandbox: `ousast plane run` writes it on the host from the memory store before the Run "
|
||||
"starts (memory.seed), so the seed did not happen; rerun through `ousast plane run`"
|
||||
)
|
||||
|
||||
|
||||
STEPS: dict[str, Callable[[Mapping[str, str], Path], dict[str, Any]]] = {
|
||||
"snapshot": snapshot, "measure": measure, "propose": propose, "improve": improve,
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None, environ: Mapping[str, str] | None = None) -> int:
|
||||
args = list(sys.argv[1:] if argv is None else argv)
|
||||
env = environ if environ is not None else os.environ
|
||||
output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None
|
||||
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None}
|
||||
here = Path.cwd()
|
||||
try:
|
||||
if output is None:
|
||||
raise ValueError("OUSAST_OUTPUT_DIR is not set")
|
||||
if len(args) != 1 or args[0] not in STEPS:
|
||||
raise ValueError(f"expected one step of {sorted(STEPS)}, got {args}")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
summary.update(step=args[0], **STEPS[args[0]](env, output))
|
||||
summary.update(status="done", units_done=1)
|
||||
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
|
||||
summary["reason"] = traceback.format_exc()[-2000:]
|
||||
finally:
|
||||
os.chdir(here)
|
||||
if output is not None:
|
||||
(output / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8")
|
||||
shown = {k: summary.get(k) for k in ("status", "step", "proposals", "accepted", "reason", "rows")}
|
||||
print(json.dumps(shown, default=str), file=sys.stderr)
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["main", "measure", "metrics", "propose", "improve", "read_records", "write_records", "write_snapshot"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -9,7 +9,8 @@ artifacts, so the reconciler and the runner stay content-blind (ai-service-plane
|
||||
Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`,
|
||||
`population`, `split` and `image` (the runner image digest the case's tasks ran on).
|
||||
|
||||
Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS`,
|
||||
Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS` (and `OUSAST_INPUT_FACTS_SUMMARY`,
|
||||
whose reused or recorded `image` the facts row carries; without one named, the row is omitted),
|
||||
`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes'
|
||||
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound
|
||||
Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from
|
||||
@@ -40,6 +41,7 @@ from .repo_facts import _digest
|
||||
|
||||
PASSES = ("a", "b", "c")
|
||||
UNKNOWN = "unknown"
|
||||
ALERT_FIELDS = ("rule_id", "rule_status", "path", "line", "function", "pin_role")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -108,8 +110,13 @@ def rows_for(
|
||||
} # fmt: skip
|
||||
rows.append(ctx.row("unit_cost", f"{label}:{unit.get('path')}", **fields))
|
||||
for alert in alerts:
|
||||
fields = {k: alert.get(k) for k in ("rule_id", "rule_status", "path", "line", "function", "pin_role")}
|
||||
rows.append(ctx.row("alert", f"{fields['rule_id']}:{fields['path']}:{fields['line']}", **fields))
|
||||
fields = {k: alert.get(k) for k in ALERT_FIELDS}
|
||||
if "in_fix_range" in alert: # the loop's `alerts` task marks it; improve/memory.py's M1 reads it
|
||||
fields["in_fix_range"] = alert["in_fix_range"]
|
||||
if alert.get("pin"): # a fixed-pin alert is a row of the pin it was raised on, not of the case's vulnerable pin
|
||||
fields["pin"] = str(alert["pin"])
|
||||
subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "")
|
||||
rows.append(ctx.row("alert", subject, **fields))
|
||||
return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows]
|
||||
|
||||
|
||||
@@ -202,6 +209,19 @@ def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||
return Path(env[name]) if env.get(name) else None
|
||||
|
||||
|
||||
def _facts_image(env: Mapping[str, str], own: str) -> str | None:
|
||||
"""The image the case's facts were computed on: without `OUSAST_INPUT_FACTS_SUMMARY` this task's own; with it,
|
||||
the reused entry's image or the image the facts task recorded, else None (unknown)."""
|
||||
path = _path(env, "OUSAST_INPUT_FACTS_SUMMARY")
|
||||
if path is None:
|
||||
return own
|
||||
summary = _json(path) or {}
|
||||
reused = summary.get("reused")
|
||||
reused = reused if isinstance(reused, dict) else {}
|
||||
image = reused.get("image") or summary.get("image")
|
||||
return image_digest(str(image)) if image else None
|
||||
|
||||
|
||||
def _units(path: Path | None) -> list[dict[str, Any]]:
|
||||
return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path)
|
||||
|
||||
@@ -226,7 +246,13 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES}
|
||||
models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES}
|
||||
agreed = _json(_path(env, "OUSAST_INPUT_AGREED"))
|
||||
rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")))
|
||||
rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")))
|
||||
facts_image = _facts_image(env, ctx.image)
|
||||
if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on
|
||||
facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split)
|
||||
rows = rows_for(facts_ctx, facts=facts, passes={}, models={}, agreed=None) + rows
|
||||
elif facts is not None:
|
||||
summary["facts_row"] = "omitted: the facts task's summary names no image, and a reuse key must not guess it"
|
||||
(output_dir / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
|
||||
if facts is not None:
|
||||
(output_dir / "facts.json").write_bytes(facts)
|
||||
|
||||
@@ -23,6 +23,8 @@ from collections.abc import Iterable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ...model.layout import is_test_path, is_vendored, layout_facts
|
||||
from ...preprocess import IGNORED_DIRS, detect_language
|
||||
|
||||
@@ -259,12 +261,13 @@ def assemble_facts(
|
||||
}
|
||||
|
||||
|
||||
def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
||||
def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | None = None, image: str | None = None) -> dict[str, Any]:
|
||||
"""Compute the facts for ``workspace`` into ``output_dir``; returns the written summary.
|
||||
|
||||
``candidates`` rows are ``{"path", "function"}`` (workspace-relative POSIX path). ``None`` means every
|
||||
function every product file defines. A `units.jsonl` row from an earlier run over the same candidate set
|
||||
is reused, not recomputed.
|
||||
is reused, not recomputed. ``image`` (the runner image, from `AX_TASK_YAML`) goes into the summary: the
|
||||
`remember` task keys the facts row by the image the facts were computed on.
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
units_path = output_dir / "units.jsonl"
|
||||
@@ -307,7 +310,8 @@ def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | No
|
||||
raise TaskFailed(str(exc)) from exc
|
||||
facts = assemble_facts(rows, wanted)
|
||||
_write_json(output_dir / "facts.json", facts)
|
||||
return write_summary(output_dir, status="done", units_done=len(rows), units_total=len(files))
|
||||
extra = {"image": image} if image else {}
|
||||
return write_summary(output_dir, status="done", units_done=len(rows), units_total=len(files), **extra)
|
||||
|
||||
|
||||
def _load_candidates(path: str | None) -> list[dict[str, Any]] | None:
|
||||
@@ -332,7 +336,9 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
workspace = Path(env.get("OUSAST_WORKSPACE_DIR") or "")
|
||||
if not env.get("OUSAST_WORKSPACE_DIR") or not workspace.is_dir():
|
||||
raise ValueError(f"OUSAST_WORKSPACE_DIR is not a directory: {workspace}")
|
||||
summary = run(workspace, output_dir, _load_candidates(env.get("OUSAST_INPUT_CANDIDATES")))
|
||||
task = yaml.safe_load(env.get("AX_TASK_YAML") or "{}")
|
||||
image = str((task.get("spec") or {}).get("image") or "") if isinstance(task, dict) else ""
|
||||
summary = run(workspace, output_dir, _load_candidates(env.get("OUSAST_INPUT_CANDIDATES")), image or None)
|
||||
except TaskFailed as exc:
|
||||
print(f"repo-facts failed: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
Reference in New Issue
Block a user