mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
bench(learn): harvest signals for the decision engine (decision-engine task 5)
`ousast plane harvest` (plane/harvest.py) writes two Runs over a label snapshot:
verify a/b + agree on the verify-family units (both sides of each labelled pair as
an opaque files-only Workspace, candidates outside the hunt's root, no declared
sites; v1 and dev-php pins with repo-facts first) and model roles on the candidate
files (pair excerpts grouped, one Task per population pin). Budgets sum under the
ceiling by construction. New roles task template; the generator re-pins it.
Spent by attribution: verify $2.54 (1961 calls, 150 units, 182 hunts per pass),
roles $1.40 (1034 calls, 799 files, 0 unclassified chunks); DeepSeek balance
19.63 -> 18.23. Engine on the pair corpus (benchmarks/learn/engine_pairs.py,
frozen e6ed785, one 3g container) is PARTIAL: 384 of 686 sides at commit, still
running; unfinished sides stay missing. benchmarks/learn/build_features.py built
1712 static+plane records into the memory store; counts-only record under
benchmarks/measurements/2026-10-01-decision-engine-harvest/.
source/roles parts are `none`, not `failed`, for a language without a declaration
pattern (C). ops/ax/README: Substrate golden-actor disk leak and the inline
Workspace limit (~20 KB).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e6ed7854bb
commit
34da916030
@@ -296,6 +296,21 @@ def _main(argv: list[str] | None) -> int:
|
||||
plane_engine.add_argument("--image", default="openultrasast:dev")
|
||||
plane_engine.add_argument("--repos", type=Path, help="the case cache (default ~/.cache/openultrasast/independent)")
|
||||
plane_engine.add_argument("--work", type=Path, help="where pins are exported and results land (default: a temporary directory)")
|
||||
plane_harvest = plane_sub.add_parser(
|
||||
"harvest", help="the decision engine's harvest Runs (verify a/b + agree, model roles) over a label snapshot"
|
||||
)
|
||||
plane_harvest.add_argument("--labels", type=Path, required=True, help="the full label rows (`ousast learn labels --out`)")
|
||||
plane_harvest.add_argument(
|
||||
"--plane", type=Path, required=True, help="where to write runs/, tasks/, workspaces/, models/, units.json (scratch)"
|
||||
)
|
||||
plane_harvest.add_argument("--name", default="harvest", help="Run name prefix: <name>-verify and <name>-roles")
|
||||
plane_harvest.add_argument("--templates", type=Path, default=Path("plane"), help="the plane root holding tasks/ and models/ templates")
|
||||
plane_harvest.add_argument("--catalog", type=Path, default=Path("benchmarks/pairs/catalog.toml"))
|
||||
plane_harvest.add_argument(
|
||||
"--cache", type=Path, default=Path.home() / ".cache" / "openultrasast", help="case clones and recipe checkouts"
|
||||
)
|
||||
plane_harvest.add_argument("--ceiling", type=float, default=10.0, help="USD: each Run's task budgets sum to at most this")
|
||||
plane_harvest.add_argument("--runner-image", type=Path, help="re-pin the task templates to the image in this file first (ops/ax/up.sh)")
|
||||
plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
|
||||
plane_ws.add_argument("population", type=Path)
|
||||
plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
|
||||
@@ -1371,6 +1386,8 @@ def _plane(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
if args.plane_command == "alerts-engine":
|
||||
return _plane_alerts_engine(args)
|
||||
if args.plane_command == "harvest":
|
||||
return _plane_harvest(args)
|
||||
if args.plane_command == "doctor":
|
||||
checks = reconciler.doctor()
|
||||
for name, ok, text in checks:
|
||||
@@ -1385,6 +1402,22 @@ def _plane(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _plane_harvest(args: argparse.Namespace) -> int:
|
||||
from .plane.generate import repin_templates
|
||||
from .plane.harvest import write_harvest
|
||||
|
||||
if args.runner_image is not None:
|
||||
repin_templates(args.templates, args.runner_image)
|
||||
command = f"ousast plane harvest --labels {args.labels.name} --name {args.name} --ceiling {args.ceiling:g}"
|
||||
files = write_harvest(
|
||||
args.labels, args.plane, name=args.name, command=command, catalog=args.catalog, cache=args.cache, templates=args.templates,
|
||||
ceiling=args.ceiling,
|
||||
) # fmt: skip
|
||||
for relative in sorted(k for k in files if k.startswith("runs/")):
|
||||
print(f"{args.plane / relative}: {files[relative].splitlines()[1].lstrip('# ')}")
|
||||
return 0
|
||||
|
||||
|
||||
def _plane_alerts_engine(args: argparse.Namespace) -> int:
|
||||
from .plane import engine_alerts
|
||||
|
||||
|
||||
Reference in New Issue
Block a user