marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

bench(learn): harvest signals for the decision engine (decision-engine task 5)

Browse Source 
`ousast plane harvest` (plane/harvest.py) writes two Runs over a label snapshot:
verify a/b + agree on the verify-family units (both sides of each labelled pair as
an opaque files-only Workspace, candidates outside the hunt's root, no declared
sites; v1 and dev-php pins with repo-facts first) and model roles on the candidate
files (pair excerpts grouped, one Task per population pin). Budgets sum under the
ceiling by construction. New roles task template; the generator re-pins it.

Spent by attribution: verify $2.54 (1961 calls, 150 units, 182 hunts per pass),
roles $1.40 (1034 calls, 799 files, 0 unclassified chunks); DeepSeek balance
19.63 -> 18.23. Engine on the pair corpus (benchmarks/learn/engine_pairs.py,
frozen e6ed785, one 3g container) is PARTIAL: 384 of 686 sides at commit, still
running; unfinished sides stay missing. benchmarks/learn/build_features.py built
1712 static+plane records into the memory store; counts-only record under
benchmarks/measurements/2026-10-01-decision-engine-harvest/.

source/roles parts are `none`, not `failed`, for a language without a declaration
pattern (C). ops/ax/README: Substrate golden-actor disk leak and the inline
Workspace limit (~20 KB).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-01 14:04:28 +02:00
co-authored by Claude Opus 5.5
parent e6ed7854bb
commit 34da916030
21 changed files with 1739 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/learn/build_features.py
+363
View File
@@ -0,0 +1,363 @@
"""Feature records for every labelled pin unit, from the harvested signals (learned-decision-engine task 5, step 3).
Host only. For each labelled pin unit of a label snapshot (``ousast learn labels --out``; conditional rows excluded),
one ``static`` and one ``plane`` record per labelled (candidate, family), built with :mod:`openultrasast.learn.features`
from:
- quick rules and deterministic roles, run here on the unit's tree: a pair side materialised from its excerpt, a
population or recipe pin exported from its clone or checkout (one at a time, deleted after, disk-guarded);
- the engine: ``benchmarks/learn/engine_pairs.py`` records for pairs, the recorded protocol scans
(``~/ousast-results/independent-v{1,2}/scans/<case>--<pin role>.json``) for populations;
- verify passes a/b and ``agree`` from the harvest Run (``harvest-verify``) and, for population v2, the recorded
``validation-46`` Run; model roles from ``harvest-roles``;
- callers and entry-point distance on a repository (``not_applicable`` on a pair excerpt: no repository exists).
An instrument that has not run for a unit yet (the engine job still running, a Run task not done) is recorded as
``failed`` with version ``missing:not-run`` -- null features, never zeros -- and counted as *missing*. Rows go to the
memory store (``OUSAST_MEMORY``, default ``~/ousast-results/plane/memory``) as kind ``features``; a rerun replaces
them by id. ``--counts`` writes the counts-only coverage record (no candidate, path or repository name).
Usage::
python benchmarks/learn/build_features.py --labels labels-<sha>.jsonl --units <harvest plane>/units.json --counts counts.json
"""
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import tempfile
from collections import Counter, defaultdict
from collections.abc import Mapping
from pathlib import Path
from typing import Any
from openultrasast.findings import quick_scan_findings
from openultrasast.learn.features import (
NONE,
NOT_APPLICABLE,
Part,
SourceIndex,
Vocabulary,
_callers_of,
_signals,
agree_part,
engine_part,
entry_distance,
entry_names,
facts_part,
function_of,
model_sinks_part,
quick_part,
record,
roles_part,
ruleset_digest,
source_part,
verify_part,
)
from openultrasast.learn.roles import RoleSet, from_model_roles, infer_for_checkout
from openultrasast.pairs import DEFAULT_CATALOG, _materialize_side, _targets, load_pair_catalog
from openultrasast.plane import memory
from openultrasast.plane.tasks.alerts import covers
from openultrasast.plane.tasks.alerts import engine_languages as _engine_languages
from openultrasast.plane.tasks.alerts import quick_languages as _quick_languages
from openultrasast.plane.tasks.repo_facts import GLOBAL
from openultrasast.preprocess import detect_language
from openultrasast.rank import rank_targets
from openultrasast.ruleset import DEFAULT_RULESET_DIR, load_ruleset
RESULTS = Path.home() / "ousast-results"
MISSING = Part("failed", version="missing:not-run")
CONDITIONAL_SOURCES = ("benign_control", "assumed_benign")
PASSES = ("a", "b", "c")
QUICK_LANGUAGES = _quick_languages()
ENGINE_LANGUAGES = _engine_languages()
def jsonl(path: Path) -> list[dict[str, Any]]:
return [json.loads(x) for x in path.read_text(encoding="utf-8").splitlines() if x.strip()] if path.is_file() else []
def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None
def done(run: Path, entry: str) -> bool:
state = (read_json(run / "state.json") or {}).get("tasks", {})
return state.get(entry, {}).get("status") == "done"
class Signals:
"""The harvested model signals: verify/agree per unit name, model roles per unit name."""
def __init__(self, verify_run: Path, roles_run: Path, v46: Path) -> None:
self.verify_run, self.roles_run, self.v46 = verify_run, roles_run, v46
self.roles: dict[str, tuple[dict[str, Any], set[str]]] = {}
state = (read_json(roles_run / "state.json") or {}).get("tasks", {})
for entry, info in state.items():
if info.get("status") != "done":
continue
payload = read_json(roles_run / entry / "roles.json") or {}
classified = {str(r["path"]) for r in jsonl(roles_run / entry / "units.jsonl") if "error" not in r}
self._index_roles(entry.removesuffix("-roles"), payload, classified)
def _index_roles(self, owner: str, payload: Mapping[str, Any], classified: set[str]) -> None:
grouped = any("/" in p and p.split("/", 1)[0].startswith("h-") for p in classified)
if not grouped: # a pin's Task: the paths are the repository's
self.roles[owner] = (dict(payload), classified)
return
for unit in {p.split("/", 1)[0] for p in classified}:
prefix = unit + "/"
def strip(paths: Any, prefix: str = prefix) -> list[str]:
return [p.removeprefix(prefix) for p in paths if str(p).startswith(prefix)]
rows = [
r | {"path": r["path"].removeprefix(prefix)}
for r in payload.get("roles") or ()
if str(r.get("path", "")).startswith(prefix)
]
unit_payload = {
"model": payload.get("model"),
"roles": rows,
"unclassified_paths": strip(payload.get("unclassified_paths") or ()),
}
self.roles[unit] = (unit_payload, set(strip(classified)))
def passes(self, base: Path, entry_stem: str) -> tuple[dict[str, list], dict[str, str | None], Any] | None:
if not (done(base, f"{entry_stem}-va") and done(base, f"{entry_stem}-vb")):
return None
passes = {p: jsonl(base / f"{entry_stem}-v{p}" / "units.jsonl") for p in PASSES}
models = {p: (read_json(base / f"{entry_stem}-v{p}" / "summary.json") or {}).get("model") for p in PASSES}
final = base / f"{entry_stem}-final" / "agreed.json"
agreed = read_json(final if final.is_file() else base / f"{entry_stem}-agree" / "agreed.json")
return passes, models, agreed
def final_of(agreed: Any, candidate: str) -> str | None:
if not agreed:
return None
disputed = {str(d["candidate"]) for d in agreed.get("disputed") or []}
for row in agreed.get("candidates") or []:
if str(row["candidate"]) == candidate:
return "agreed" if row.get("agreed") else "disputed" if candidate in disputed else "rejected"
return None
def plane_parts(signals: Signals, stems: list[tuple[Path, str]], roles_owner: str | None, candidate: str, line: int,
path: str, function: str,
verify_expected: bool) -> dict[str, Part]: # fmt: skip
"""verify, agree, model_sinks of one candidate: ``stems`` are (run dir, entry stem) that may have asked it."""
parts: dict[str, Part] = {"verify": NONE, "agree": NONE}
pending = False
for base, stem in stems:
got = signals.passes(base, stem)
if got is None:
pending = pending or (base == signals.verify_run and verify_expected)
continue
passes, models, agreed = got
part = verify_part(passes, candidate, line, "+".join(sorted({m for m in models.values() if m})) or None)
if part.state == "ran":
parts = {"verify": part, "agree": agree_part(final_of(agreed, candidate))}
break
else:
if pending:
parts = {"verify": MISSING, "agree": MISSING}
if roles_owner is None or roles_owner not in signals.roles:
parts["model_sinks"] = MISSING
else:
payload, classified = signals.roles[roles_owner]
if path not in classified:
parts["model_sinks"] = MISSING
else:
parts["model_sinks"] = model_sinks_part(from_model_roles(payload), path, function, str(payload.get("model") or "") or None)
return parts
def static_parts(root: Path, index: SourceIndex, quick: Mapping, engine: Mapping, engine_state: Part | None, engine_result: Mapping | None,
inferred: RoleSet, rules: Mapping, quick_version: str, path: str, function: str, family: str, repository: bool,
entries: tuple[set[str], set[str]] | None) -> dict[str, Part]: # fmt: skip
language = detect_language(Path(path)) or "other"
key = (path, function, family)
parts: dict[str, Part] = {}
parts["quick"] = quick_part(quick.get(key, []), rules, quick_version) if covers(QUICK_LANGUAGES, language) else NONE
if language not in ENGINE_LANGUAGES:
parts["engine"] = NONE
elif engine_state is not None:
parts["engine"] = engine_state
else:
result = engine_result or {}
questions, completed = int(result.get("questions") or 0), int(result.get("completed") or 0)
parts["engine"] = engine_part(
engine.get(key, []), completion=completed / questions if questions else None, degraded=bool(result.get("degradations")),
vocabulary=Vocabulary.load(language), version=str(result.get("image") or "recorded"),
) # fmt: skip
lines = index.lines(path)
parts["source"] = source_part(lines, language, function)
parts["roles"] = roles_part(function_of(lines, path, language, function), inferred, language)
parts["delta"] = NOT_APPLICABLE
if repository and entries is not None:
distance = 0 if function == GLOBAL and path in entries[1] else entry_distance(function, entries[0], index.callers())
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
parts["facts"] = facts_part(_callers_of(index, index.callers(), path, function), function, "host")
else:
parts["entry_points"] = NOT_APPLICABLE
parts["facts"] = NOT_APPLICABLE
return parts
def engine_for(result: Mapping[str, Any] | None) -> Part | None:
"""None when the engine ran (its findings feed engine_part); else the state Part."""
if result is None:
return MISSING
state = result.get("state", "ran" if result.get("questions") else "failed")
if state == "ran" and int(result.get("questions") or 0) > 0:
return None
if state == "none":
return NONE
return Part("failed", version=str(result.get("image") or "recorded"))
def export(clone: Path, pin: str, target: Path) -> bool:
target.mkdir(parents=True)
archive = target.parent / "pin.tar"
with archive.open("wb") as stream:
ok = subprocess.run(["git", "-C", str(clone), "archive", pin], stdout=stream, check=False).returncode == 0
ok = ok and subprocess.run(["tar", "-xf", str(archive), "-C", str(target)], check=False).returncode == 0
archive.unlink(missing_ok=True)
return ok and any(target.iterdir())
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--labels", type=Path, required=True)
parser.add_argument("--units", type=Path, required=True, help="units.json of the harvest plane")
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--cache", type=Path, default=Path.home() / ".cache" / "openultrasast")
parser.add_argument("--engine", type=Path, default=RESULTS / "plane" / "harvest-engine")
parser.add_argument("--verify-run", type=Path, default=RESULTS / "plane" / "harvest-verify")
parser.add_argument("--roles-run", type=Path, default=RESULTS / "plane" / "harvest-roles")
parser.add_argument("--v46", type=Path, default=RESULTS / "plane" / "validation-46")
parser.add_argument("--scratch", type=Path, required=True)
parser.add_argument("--min-free-gb", type=float, default=1.2)
parser.add_argument("--counts", type=Path, required=True)
parser.add_argument("--store", help="OUSAST_MEMORY spec (default: the environment's)")
parser.add_argument("--limit", type=int, default=0, help="stop after this many targets (a test)")
parser.add_argument("--kinds", default="pairs,pin", help="which targets: pairs, pin (population and recipe pins)")
args = parser.parse_args()
labels = [r for r in jsonl(args.labels) if r["unit"] == "pin" and r["source"] not in CONDITIONAL_SOURCES and not r.get("conditional")]
units = json.loads(args.units.read_text(encoding="utf-8"))
by_pair = {(u["ref"], u["side"]): n for n, u in units.items() if u["source"] == "pairs"}
by_pin: dict[tuple[str, str], list[str]] = defaultdict(list)
for n, u in units.items():
if u["source"] != "pairs":
by_pin[(u["repo"], u["pin"])].append(n)
targets: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
for row in labels:
key = ("pairs", row["source_ref"], row["pin_role"]) if row["source"] == "pairs" else ("pin", row["repo"], row["pin"])
targets[key].append(row)
cases = {c.name: c for c in load_pair_catalog(args.catalog)}
loaded = load_ruleset(DEFAULT_RULESET_DIR)
rules = {r.rule_id: r for r in loaded}
quick_version = ruleset_digest(loaded)
signals = Signals(args.verify_run, args.roles_run, args.v46)
store = memory.open_store(args.store)
tally: Counter[tuple[str, ...]] = Counter()
coverage: dict[str, Counter[str]] = defaultdict(Counter)
skipped: Counter[str] = Counter()
args.scratch.mkdir(parents=True, exist_ok=True)
chosen = [(k, r) for k, r in sorted(targets.items()) if k[0] in args.kinds.split(",")]
for number, ((kind, ref, role_or_pin), rows) in enumerate(chosen):
if args.limit and number >= args.limit:
break
with tempfile.TemporaryDirectory(dir=args.scratch) as tmp:
root = Path(tmp) / "tree"
if kind == "pairs":
name = by_pair.get((ref, role_or_pin))
if name is None or ref not in cases:
skipped["pair side not in the harvest units"] += len(rows)
continue
unit = units[name]
_materialize_side(root, cases[ref], side="vuln" if role_or_pin == "vulnerable" else "fixed")
engine_result = read_json(args.engine / f"{ref}--{'vuln' if role_or_pin == 'vulnerable' else 'fixed'}.json")
stems = [(args.verify_run, name)]
owner, repository, repo, pin = name, False, unit["repo"], unit["pin"]
else:
names = by_pin.get((ref, role_or_pin), [])
if not names:
skipped["pin not in the harvest units"] += len(rows)
continue
unit = units[names[0]]
repo, pin, repository = ref, role_or_pin, True
if shutil.disk_usage(args.scratch).free / 1e9 < args.min_free_gb:
skipped["disk guard: pin not exported"] += len(rows)
continue
if unit["source"] == "dev-php":
source_dir = args.cache / "repos" / unit["ref"].split(":")[0] / pin[:12]
shutil.copytree(source_dir, root, symlinks=True, ignore=shutil.ignore_patterns(".git"))
engine_result = None
else:
case_id = unit["ref"]
if not export(args.cache / "independent" / case_id, pin, root):
skipped["pin export failed"] += len(rows)
continue
split = {r["split"] for r in rows} | {unit["source"]}
population = "independent-v1" if "population-v1" in split else "independent-v2"
engine_result = read_json(RESULTS / population / "scans" / f"{case_id}--{rows[0]['pin_role']}.json")
stems = [(args.verify_run, n) for n in names] + [(args.v46, unit["ref"])]
owner = next(
(n for n in names if f"{n}-roles" in {e for e in (read_json(args.roles_run / "state.json") or {}).get("tasks", {})}),
names[0],
)
index = SourceIndex(root)
targets_ = _targets(root)
findings = quick_scan_findings(root, targets_, rank_targets(targets_), None)
engine_state = engine_for(engine_result)
quick, engine = _signals(index, findings, engine_result if engine_state is None else None, rules)
inferred = infer_for_checkout(root)
entries = entry_names(root, {r["candidate"].partition("::")[0] for r in rows}) if repository else None
out_rows: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
for row in rows:
candidate, family = row["candidate"], row["family"]
if (candidate, family) in seen:
continue
seen.add((candidate, family))
path, _, function = candidate.partition("::")
cand_line = next((c[2] for n in ([owner] if kind == "pairs" else by_pin[(repo, pin)]) for c in units[n]["candidates"]
if f"{c[0]}::{c[1]}" == candidate), None) # fmt: skip
parts = static_parts(root, index, quick, engine, engine_state, engine_result, inferred, rules, quick_version, path,
function, family, repository, entries) # fmt: skip
language = detect_language(Path(path)) or "other"
verify_expected = any(units[n]["family"] == family for n in ([owner] if kind == "pairs" else by_pin[(repo, pin)]))
plane = {**parts, **plane_parts(signals, stems, owner, candidate, cand_line or 1, path, function, verify_expected)}
for profile, use in (("static", parts), ("plane", plane)):
rec = record(candidate, family, language, profile, use)
out_rows.append({
"id": memory.row_id("features", "harvest", owner, f"{candidate}:{family}:{profile}"), "kind": "features",
"repo": memory.repo_key(repo), "pin": pin, "run": "harvest", "task": owner, "population": row["source"],
"split": row["split"], "image": str((engine_result or {}).get("image") or "none"), **rec,
}) # fmt: skip
tally[(profile, family)] += 1
if profile == "plane":
for inst, info in rec["instruments"].items():
state = "missing" if info.get("version") == "missing:not-run" else info["state"]
coverage[f"{family}:{inst}"][state] += 1
store.ingest_rows("harvest", f"features:{owner}", out_rows)
print(json.dumps({"target": owner, "rows": len(out_rows)}), flush=True)
counts = {
"records": {f"{p}:{f}": n for (p, f), n in sorted(tally.items())},
"instrument_states_plane": {k: dict(sorted(v.items())) for k, v in sorted(coverage.items())},
"skipped_label_rows": dict(skipped),
"store": store.describe(),
}
args.counts.write_text(json.dumps(counts, indent=1, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"records": sum(tally.values()), "skipped": dict(skipped)}))
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/learn/engine_pairs.py
+181
View File
@@ -0,0 +1,181 @@
"""Engine signals for the labelled pair corpus, on the host (learned-decision-engine task 5, design "Cost on the 7 GB host").
Per labelled pair (``ousast learn labels`` rows of source ``pairs``) and side (``vuln``, ``fixed``): the side's
excerpt -- with the documents that register it -- is materialised into a temporary directory, and the Joern engine
runs on it in ``openultrasast:dev`` (``benchmarks/push/finding_dump.py`` from a FROZEN source export, ``--network
none --memory 3g``, the pair's labelled families, ONE container at a time). The result is one JSON per side under
``--out`` (``<pair>--<side>.json``), keyed by the side's content sha256 and the engine image id; a rerun skips sides
already recorded, so the job resumes where it stopped.
The instrument must have read its input (:func:`openultrasast.plane.engine_alerts.read_record`): no result file, no
``read N files`` line or ``questions == 0`` is recorded as ``state: failed`` with the reason (retried once), never as
zero findings. A side whose language the engine has no model for is ``state: none`` and is not run.
Guards: free disk is checked before every side and every 60 s while a container runs; below ``--min-free-gb`` the
container is killed by its name, the side is left unrecorded (``missing``) and the job stops. A file ``STOP`` in
``--out`` stops the job cleanly between sides. Progress lines go to ``--out/progress.jsonl``.
Usage::
python benchmarks/learn/engine_pairs.py --labels labels-<sha>.jsonl --source <frozen export> --out <dir>
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shutil
import subprocess
import sys
import tempfile
import threading
from pathlib import Path
from openultrasast.pairs import DEFAULT_CATALOG, _materialize_side, load_pair_catalog
from openultrasast.plane.engine_alerts import EngineAlertsError, docker_command, frozen_source, read_record
from openultrasast.plane.tasks.alerts import engine_languages
from openultrasast.preprocess import detect_language
SIDES = ("vuln", "fixed")
_OFFERED = re.compile(r"^read (\d+) files, \d+ bytes; (\d+) regions offer", re.MULTILINE)
def free_gb(path: Path) -> float:
return shutil.disk_usage(path).free / 1e9
def units(labels: Path, catalog: Path) -> list[tuple[object, str, tuple[str, ...]]]:
"""(pair case, side, families) for every labelled pair, in name order."""
families: dict[str, set[str]] = {}
for line in labels.read_text(encoding="utf-8").splitlines():
row = json.loads(line)
if row["source"] == "pairs" and row["unit"] == "pin" and row["family"] != "unknown":
families.setdefault(row["source_ref"], set()).add(row["family"])
cases = {case.name: case for case in load_pair_catalog(catalog)}
return [(cases[name], side, tuple(sorted(families[name]))) for name in sorted(families) for side in SIDES]
def image_id(image: str) -> str:
done = subprocess.run(["docker", "image", "inspect", "--format", "{{.Id}}", image], capture_output=True, text=True, check=False)
if done.returncode != 0 or not done.stdout.strip():
raise SystemExit(f"no engine image {image}")
return done.stdout.strip()
def run_side(
command: list[str], name: str, timeout: float, out: Path, min_free: float
) -> tuple[subprocess.CompletedProcess[str] | None, str]:
"""Run one container; a watchdog kills it by name when free disk drops below ``min_free`` GB."""
tripped: list[str] = []
done = threading.Event()
def watch() -> None:
while not done.wait(60.0):
if free_gb(out) < min_free:
tripped.append(f"free disk {free_gb(out):.2f} GB < {min_free} GB")
subprocess.run(["docker", "kill", name], capture_output=True, check=False)
return
watcher = threading.Thread(target=watch, daemon=True)
watcher.start()
try:
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False)
except subprocess.TimeoutExpired:
subprocess.run(["docker", "kill", name], capture_output=True, check=False)
completed = None
done.set()
return completed, (tripped[0] if tripped else "")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--labels", type=Path, required=True)
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--source", type=Path, required=True, help="frozen export: git archive HEAD src benchmarks/push/finding_dump.py")
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--image", default="openultrasast:dev")
parser.add_argument("--deadline", type=float, default=600.0)
parser.add_argument("--min-free-gb", type=float, default=1.0)
parser.add_argument("--limit", type=int, default=0, help="stop after this many sides were run (0: all)")
args = parser.parse_args()
source = frozen_source(args.source)
args.out.mkdir(parents=True, exist_ok=True)
image = image_id(args.image)
covered = engine_languages()
progress = (args.out / "progress.jsonl").open("a", encoding="utf-8")
ran = 0
for case, side, families in units(args.labels, args.catalog):
result_path = args.out / f"{case.name}--{side}.json"
if result_path.is_file():
continue
if (args.out / "STOP").exists() or (args.limit and ran >= args.limit):
print("stopped between sides", flush=True)
return 0
if free_gb(args.out) < args.min_free_gb:
print(f"disk guard: {free_gb(args.out):.2f} GB free; stopping", flush=True)
return 4
excerpt = case.vuln_file if side == "vuln" else case.fixed_file
if not excerpt.is_file():
entry = {"state": "failed", "reason": "excerpt not materialised"}
else:
entry = {"sha256": hashlib.sha256(excerpt.read_bytes()).hexdigest()}
language = detect_language(Path(case.relpath)) or "other"
if language not in covered:
entry.update(state="none", reason=f"no engine model for {language}")
else:
entry.update(_scan(case, side, families, source, args, image))
ran += 1
if entry.get("state") == "missing":
print(json.dumps({"pair": case.name, "side": side, **entry}), flush=True)
return 4
record = {"pair": case.name, "side": side, "families": list(families), "image": image, **entry}
result_path.write_text(json.dumps(record, indent=1, sort_keys=True) + "\n", encoding="utf-8")
line = {k: record.get(k) for k in ("pair", "side", "state", "questions", "completed", "seconds", "reason")}
line["findings"] = len(record.get("findings") or [])
progress.write(json.dumps(line) + "\n")
progress.flush()
print(json.dumps(line), flush=True)
return 0
def _scan(case: object, side: str, families: tuple[str, ...], source: Path, args: argparse.Namespace, image: str) -> dict:
reason = ""
for attempt in (1, 2):
with tempfile.TemporaryDirectory(prefix="ep-", dir=args.out) as tmp:
checkout = _materialize_side(Path(tmp) / "case", case, side=side)
out_dir = Path(tmp) / "out"
out_dir.mkdir()
name = f"ousast-ep-{hashlib.sha1(f'{case.name}{side}'.encode()).hexdigest()[:12]}"
command = docker_command(source, checkout, out_dir, "result", ",".join(families), args.deadline, args.image)
command[2:2] = ["--name", name]
completed, tripped = run_side(command, name, args.deadline + 300, args.out, args.min_free_gb)
if tripped:
return {"state": "missing", "reason": tripped}
if completed is None:
reason = f"timeout after {args.deadline + 300:.0f}s"
continue
log = (completed.stdout or "") + "\n" + (completed.stderr or "")
try:
record = read_record(out_dir / "result.json", log, completed.returncode)
except EngineAlertsError as exc:
reason = f"attempt {attempt}: {exc}"[:600]
offered = _OFFERED.search(log)
result = out_dir / "result.json"
if offered is not None and int(offered.group(1)) > 0 and result.is_file():
# It read its input and finished without asking: deterministic (a retry asks nothing either). Still
# `failed` by the design's rule (questions == 0 is never zero findings); the reason keeps it apart
# from a launch failure, because on a fixed side "no question" may mean "no sink left".
recorded = json.loads(result.read_text(encoding="utf-8"))
regions = int(offered.group(2))
why = "no region offers the families" if regions == 0 else f"no question asked for {regions} regions"
return {"state": "failed", "reason": why, "files": int(offered.group(1)), "regions": regions,
"seconds": recorded.get("seconds"), "degradations": recorded.get("degradations")} # fmt: skip
continue
return {"state": "ran", **{k: v for k, v in record.items() if k != "root"}}
return {"state": "failed", "reason": reason}
if __name__ == "__main__":
sys.exit(main())
benchmarks/measurements/2026-09-08-module-audit.json
+26 -5
View File
@@ -1,7 +1,7 @@
{
"modules": 143,
"modules": 144,
"counts": {
"load_bearing": 136,
"load_bearing": 137,
"standalone_capability": 7
},
"orphaned": [],
@@ -354,6 +354,7 @@
"cli",
"learn.features",
"learn.roles",
"plane.harvest",
"plane.tasks.features",
"plane.tasks.remember",
"plane.tasks.roles"
@@ -372,13 +373,14 @@
"classification": "load_bearing",
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
"importers": [
"cli"
"cli",
"plane.harvest"
]
},
{
"module": "learn.roles",
"classification": "load_bearing",
"reason": "imported by learn.features, plane.tasks.features, plane.tasks.roles",
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
"importers": [
"learn.features",
"plane.tasks.features",
@@ -709,6 +711,7 @@
"learn.labels",
"model.candidates",
"pair_gate",
"plane.harvest",
"plane.tasks.loop"
]
},
@@ -722,6 +725,7 @@
"plane.egress",
"plane.engine_alerts",
"plane.generate",
"plane.harvest",
"plane.memory",
"plane.reconciler",
"plane.router",
@@ -769,6 +773,15 @@
"module": "plane.generate",
"classification": "load_bearing",
"reason": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set",
"importers": [
"cli",
"plane.harvest"
]
},
{
"module": "plane.harvest",
"classification": "load_bearing",
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
"importers": [
"cli"
]
@@ -780,6 +793,7 @@
"importers": [
"plane.egress",
"plane.generate",
"plane.harvest",
"plane.reconciler",
"plane.router",
"plane.runner",
@@ -803,6 +817,7 @@
"reason": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`",
"importers": [
"plane.generate",
"plane.harvest",
"plane.memory",
"plane.workspaces"
]
@@ -833,6 +848,7 @@
"learn.roles",
"plane.engine_alerts",
"plane.generate",
"plane.harvest",
"plane.memory",
"plane.tasks.agree",
"plane.tasks.alerts",
@@ -888,6 +904,7 @@
"learn.labels",
"learn.roles",
"plane.generate",
"plane.harvest",
"plane.tasks.alerts",
"plane.tasks.features",
"plane.tasks.remember",
@@ -898,7 +915,9 @@
"module": "plane.tasks.roles",
"classification": "load_bearing",
"reason": "learned-decision-engine Req 8.1: model role classification per file chunk (model_sinks.py moved in); its Run template comes with the harvest (task 5)",
"importers": []
"importers": [
"plane.harvest"
]
},
{
"module": "plane.tasks.verify",
@@ -906,6 +925,7 @@
"reason": "ai-service-plane Req 5.3/2.1-2.3/7.2: the batched hunt with known callers, metered and resumable per file hunt",
"importers": [
"plane.generate",
"plane.harvest",
"plane.tasks.agree",
"plane.tasks.roles"
]
@@ -966,6 +986,7 @@
"model.partitions",
"model.scan",
"pairs",
"plane.harvest",
"plane.tasks.alerts",
"plane.tasks.features",
"plane.tasks.repo_facts",
benchmarks/measurements/2026-10-01-decision-engine-harvest/record.json
+477
View File
@@ -0,0 +1,477 @@
{
"balance_usd": {
"after_roles": 18.23,
"after_verify": 18.75,
"before": 19.63
},
"ceilings_usd": {
"roles": 10.0,
"verify": 10.0
},
"created": "2026-10-01",
"engine": {
"frozen_source_commit": "e6ed7854bb8ef17d6a4c39971ea253b4e385af23",
"image": "openultrasast:dev sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"pair_sides_total": 686,
"population_engine": "recorded protocol scans of independent-v1/v2 (vulnerable and fixed pins); dev-php not run (missing)",
"sides_missing": 302,
"sides_recorded": {
"failed (read its input, asked no question)": 286,
"none": 6,
"ran": 92
},
"status": "running in the background at commit time; unfinished sides are `missing` in the records, never zero"
},
"features": {
"instrument_states_plane": {
"access_control:agree": {
"none": 26
},
"access_control:delta": {
"not_applicable": 26
},
"access_control:engine": {
"missing": 6,
"ran": 20
},
"access_control:entry_points": {
"not_applicable": 26
},
"access_control:facts": {
"not_applicable": 26
},
"access_control:language": {
"ran": 26
},
"access_control:model_sinks": {
"ran": 26
},
"access_control:quick": {
"ran": 26
},
"access_control:roles": {
"failed": 13,
"ran": 13
},
"access_control:source": {
"failed": 13,
"ran": 13
},
"access_control:verify": {
"none": 26
},
"config_secrets:agree": {
"none": 4,
"ran": 31
},
"config_secrets:delta": {
"not_applicable": 35
},
"config_secrets:engine": {
"missing": 4,
"none": 23,
"ran": 8
},
"config_secrets:entry_points": {
"not_applicable": 8,
"ran": 27
},
"config_secrets:facts": {
"not_applicable": 8,
"ran": 27
},
"config_secrets:language": {
"ran": 35
},
"config_secrets:model_sinks": {
"ran": 35
},
"config_secrets:quick": {
"ran": 35
},
"config_secrets:roles": {
"failed": 14,
"ran": 21
},
"config_secrets:source": {
"failed": 4,
"ran": 31
},
"config_secrets:verify": {
"none": 4,
"ran": 31
},
"deserialization:agree": {
"none": 40
},
"deserialization:delta": {
"not_applicable": 40
},
"deserialization:engine": {
"failed": 21,
"ran": 19
},
"deserialization:entry_points": {
"not_applicable": 40
},
"deserialization:facts": {
"not_applicable": 40
},
"deserialization:language": {
"ran": 40
},
"deserialization:model_sinks": {
"ran": 40
},
"deserialization:quick": {
"ran": 40
},
"deserialization:roles": {
"failed": 20,
"ran": 20
},
"deserialization:source": {
"failed": 20,
"ran": 20
},
"deserialization:verify": {
"none": 40
},
"injection:agree": {
"missing": 4,
"none": 25,
"ran": 139
},
"injection:delta": {
"not_applicable": 168
},
"injection:engine": {
"failed": 22,
"missing": 78,
"none": 16,
"ran": 52
},
"injection:entry_points": {
"not_applicable": 70,
"ran": 98
},
"injection:facts": {
"not_applicable": 70,
"ran": 98
},
"injection:language": {
"ran": 168
},
"injection:model_sinks": {
"missing": 4,
"ran": 164
},
"injection:quick": {
"none": 2,
"ran": 166
},
"injection:roles": {
"failed": 44,
"none": 2,
"ran": 122
},
"injection:source": {
"failed": 36,
"none": 2,
"ran": 130
},
"injection:verify": {
"missing": 4,
"none": 25,
"ran": 139
},
"memory:agree": {
"none": 346
},
"memory:delta": {
"not_applicable": 346
},
"memory:engine": {
"failed": 116,
"missing": 218,
"none": 6,
"ran": 6
},
"memory:entry_points": {
"not_applicable": 346
},
"memory:facts": {
"not_applicable": 346
},
"memory:language": {
"ran": 346
},
"memory:model_sinks": {
"missing": 4,
"ran": 342
},
"memory:quick": {
"none": 4,
"ran": 342
},
"memory:roles": {
"none": 346
},
"memory:source": {
"none": 346
},
"memory:verify": {
"none": 346
},
"output_encoding:agree": {
"none": 112
},
"output_encoding:delta": {
"not_applicable": 112
},
"output_encoding:engine": {
"failed": 24,
"missing": 82,
"ran": 6
},
"output_encoding:entry_points": {
"not_applicable": 112
},
"output_encoding:facts": {
"not_applicable": 112
},
"output_encoding:language": {
"ran": 112
},
"output_encoding:model_sinks": {
"ran": 112
},
"output_encoding:quick": {
"ran": 112
},
"output_encoding:roles": {
"failed": 16,
"ran": 96
},
"output_encoding:source": {
"failed": 16,
"ran": 96
},
"output_encoding:verify": {
"none": 112
},
"path:agree": {
"none": 29
},
"path:delta": {
"not_applicable": 29
},
"path:engine": {
"failed": 22,
"missing": 5,
"ran": 2
},
"path:entry_points": {
"not_applicable": 28,
"ran": 1
},
"path:facts": {
"not_applicable": 28,
"ran": 1
},
"path:language": {
"ran": 29
},
"path:model_sinks": {
"ran": 29
},
"path:quick": {
"ran": 29
},
"path:roles": {
"failed": 15,
"ran": 14
},
"path:source": {
"failed": 15,
"ran": 14
},
"path:verify": {
"none": 29
},
"unknown:agree": {
"none": 4
},
"unknown:delta": {
"not_applicable": 4
},
"unknown:engine": {
"missing": 2,
"none": 2
},
"unknown:entry_points": {
"not_applicable": 4
},
"unknown:facts": {
"not_applicable": 4
},
"unknown:language": {
"ran": 4
},
"unknown:model_sinks": {
"ran": 4
},
"unknown:quick": {
"ran": 4
},
"unknown:roles": {
"none": 4
},
"unknown:source": {
"none": 4
},
"unknown:verify": {
"none": 4
},
"untrusted_destination:agree": {
"none": 11,
"ran": 85
},
"untrusted_destination:delta": {
"not_applicable": 96
},
"untrusted_destination:engine": {
"failed": 38,
"missing": 12,
"none": 17,
"ran": 29
},
"untrusted_destination:entry_points": {
"not_applicable": 52,
"ran": 44
},
"untrusted_destination:facts": {
"not_applicable": 52,
"ran": 44
},
"untrusted_destination:language": {
"ran": 96
},
"untrusted_destination:model_sinks": {
"ran": 96
},
"untrusted_destination:quick": {
"ran": 96
},
"untrusted_destination:roles": {
"failed": 39,
"ran": 57
},
"untrusted_destination:source": {
"failed": 33,
"ran": 63
},
"untrusted_destination:verify": {
"none": 11,
"ran": 85
}
},
"records": {
"plane:access_control": 26,
"plane:config_secrets": 35,
"plane:deserialization": 40,
"plane:injection": 168,
"plane:memory": 346,
"plane:output_encoding": 112,
"plane:path": 29,
"plane:unknown": 4,
"plane:untrusted_destination": 96,
"static:access_control": 26,
"static:config_secrets": 35,
"static:deserialization": 40,
"static:injection": 168,
"static:memory": 346,
"static:output_encoding": 112,
"static:path": 29,
"static:unknown": 4,
"static:untrusted_destination": 96
},
"skipped_label_rows": {},
"store": "file://~/ousast-results/plane/memory (kind features, run harvest)"
},
"host_peaks": {
"disk_free_mb_min": 960,
"disk_incident": "Substrate kept every golden actor (~24 MB each, deletion failing on a missing snapshot bucket); free disk fell to 0.95 GB, runs were stopped, the bucket created and orphaned actor directories removed",
"mem_available_mb_min": 1990
},
"label_snapshot_sha256": "18337014d8a5c8df67e06f4e9d8e72c1fb8f82b1ed1b9a18836ab37e6cf4fdc9",
"note": "Harvest of the learned decision engine (task 5), COUNTS ONLY. Spend is the plane's attribution table (`ousast plane status`, task meters priced at plane/models/deepseek-flash.yaml); the DeepSeek account moved less (balance below). Feature records are in the local memory store (kind `features`, run `harvest`); the label rows, the units index and the manifests name functions and stay local.",
"roles": {
"attribution": {
"cache_hit_tokens": null,
"calls": 1034,
"output_tokens": null,
"prompt_tokens": null,
"usd": 1.4036
},
"cache_hit_tokens": 329600,
"chunks": 1034,
"files": 799,
"output_tokens": 370155,
"prompt_tokens": 2398669,
"task_budget_sum_usd": 4.917,
"tasks_done": 151,
"unclassified_chunks": 0
},
"runner_image": "localhost:5001/ousast-runner@sha256:412db0a1504af8f1b23a395af8d1dcbd152c1bb6d16594f2b733067163a31884",
"verify": {
"attribution": {
"cache_hit_tokens": 6275060,
"calls": 1961,
"output_tokens": 306941,
"prompt_tokens": 10921167,
"usd": 2.5373
},
"file_hunts_done": 364,
"flagged_by_pass_units": {
"dev-php:vulnerable": {
"both": 2
},
"pairs:fixed": {
"both": 4,
"none": 58,
"one": 1
},
"pairs:vulnerable": {
"both": 45,
"none": 11,
"one": 7
},
"population-v1:fixed": {
"both": 5,
"none": 2,
"one": 4
},
"population-v1:vulnerable": {
"both": 8,
"none": 2,
"one": 1
}
},
"hunts_per_pass": 182,
"infrastructure_failures": "ax apply / actor template not found for Workspaces over the inline limit (units then skipped); two tasks interrupted by a disk stop were re-run",
"skipped_units": {
"pairs": 4,
"reason": "excerpt over the 20 KB inline Workspace limit"
},
"task_budget_sum_usd": 8.624,
"tasks_done": 474,
"units": {
"dev-php:vulnerable": 2,
"pairs:fixed": 63,
"pairs:vulnerable": 63,
"population-v1:fixed": 11,
"population-v1:vulnerable": 11
}
}
}