bench(learn): harvest signals for the decision engine (decision-engine task 5)

`ousast plane harvest` (plane/harvest.py) writes two Runs over a label snapshot: verify a/b + agree on the verify-family units (both sides of each labelled pair as an opaque files-only Workspace, candidates outside the hunt's root, no declared sites; v1 and dev-php pins with repo-facts first) and model roles on the candidate files (pair excerpts grouped, one Task per population pin). Budgets sum under the ceiling by construction. New roles task template; the generator re-pins it. Spent by attribution: verify $2.54 (1961 calls, 150 units, 182 hunts per pass), roles $1.40 (1034 calls, 799 files, 0 unclassified chunks); DeepSeek balance 19.63 -> 18.23. Engine on the pair corpus (benchmarks/learn/engine_pairs.py, frozen e6ed785, one 3g container) is PARTIAL: 384 of 686 sides at commit, still running; unfinished sides stay missing. benchmarks/learn/build_features.py built 1712 static+plane records into the memory store; counts-only record under benchmarks/measurements/2026-10-01-decision-engine-harvest/. source/roles parts are `none`, not `failed`, for a language without a declaration pattern (C). ops/ax/README: Substrate golden-actor disk leak and the inline Workspace limit (~20 KB). Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
2026-10-16 03:49:26 +00:00 · 2026-10-01 14:04:28 +02:00
parent e6ed7854bb
commit 34da916030
21 changed files with 1739 additions and 137 deletions
@@ -156,6 +156,18 @@ def test_roles_feature_part(tmp_path: Path) -> None:
    assert [f.name for f in functions_of("a.php", lines, "php")][:2] == ["run_sql", "fetch_rows"]


+def test_a_language_without_declarations_is_none_not_failed() -> None:
+    """C has no declaration pattern: the span and wrapper roles cannot run there (no coverage), which is not a broken
+    instrument (harvest 2026-10-01: every memory pair read as `failed`)."""
+    from openultrasast.learn.features import source_part
+    from openultrasast.learn.roles import RoleSet
+
+    lines = ["int f(char *s) {", "  return strcpy(buf, s);", "}"]
+    assert source_part(lines, "c", "f").state == "none"
+    assert roles_part(None, RoleSet(sources=("inferred_wrapper",)), "c").state == "none"
+    assert source_part(["def g():", "    pass"], "python", "missing").state == "failed"
+
+
 def test_build_for_scan_records_the_roles_instrument(tmp_path: Path) -> None:
    from openultrasast.findings import quick_scan_findings
    from openultrasast.preprocess import preprocess_repository
@@ -0,0 +1,100 @@
+"""The decision engine's harvest Runs (learned-decision-engine task 5): `ousast plane harvest`'s generator."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+import yaml
+
+from openultrasast.plane import reconciler
+from openultrasast.plane.harvest import MAX_INLINE_BYTES, Unit, blob_sha1, declaration_line, render_harvest
+from openultrasast.plane.manifests import load_manifests
+
+PLANE = Path("plane")
+PIN = "0123456789abcdef0123456789abcdef01234567"
+
+
+def _templates() -> dict:
+    return load_manifests([PLANE / "tasks" / f"{n}.yaml" for n in ("repo-facts", "verify", "agree", "roles")]).tasks
+
+
+def _units() -> list[Unit]:
+    code = "import os\n\ndef run(cmd):\n    os.system(cmd)\n"
+    excerpt = Unit("h-aaaaaaaaaaaa", "pairs", "x-cwe-078-os-system", "vulnerable", "pairs/github/x", blob_sha1(code), "injection",
+                   (("app.py", "run", 3),), (("app.py", code),))  # fmt: skip
+    roles_only = Unit("h-bbbbbbbbbbbb", "pairs", "y-cwe-416", "fixed", "pairs/vfc/y", blob_sha1("int f(){}\n"), "",
+                      (("f.c", "f", 1),), (("f.c", "int f(){}\n"),))  # fmt: skip
+    big = "x = 1\n" * (MAX_INLINE_BYTES // 6 + 10)
+    oversized = Unit(
+        "h-cccccccccccc",
+        "pairs",
+        "z",
+        "vulnerable",
+        "pairs/github/z",
+        blob_sha1(big),
+        "injection",
+        (("big.py", GLOBAL_, 1),),
+        (("big.py", big),),
+    )
+    repo = Unit("h-dddddddddddd", "population-v1", "case", "vulnerable", "https://github.com/o/r", PIN, "injection",
+                (("a.py", "q", 10), ("b.py", "r", 20)), sizes=(("a.py", 4000, 100), ("b.py", 8000, 900)))  # fmt: skip
+    return [excerpt, roles_only, oversized, repo]
+
+
+GLOBAL_ = "<global>"
+
+
+def _render(tmp_path: Path, ceiling: float = 10.0) -> tuple[dict[str, str], Path]:
+    files = render_harvest(_units(), _templates(), "hv", "test", ceiling=ceiling)
+    for rel, text in files.items():
+        (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)
+        (tmp_path / rel).write_text(text)
+    (tmp_path / "models").mkdir()
+    (tmp_path / "models" / "deepseek-flash.yaml").write_text((PLANE / "models" / "deepseek-flash.yaml").read_text())
+    return files, tmp_path
+
+
+def test_verify_run_shape_and_ceiling(tmp_path: Path) -> None:
+    files, root = _render(tmp_path, ceiling=0.05)
+    run, manifests = reconciler.load_run(root / "runs" / "hv-verify.yaml")
+    names = [t.name for t in run.tasks]
+    assert names == [
+        "h-aaaaaaaaaaaa-va", "h-aaaaaaaaaaaa-vb", "h-aaaaaaaaaaaa-agree",
+        "h-dddddddddddd-facts", "h-dddddddddddd-va", "h-dddddddddddd-vb", "h-dddddddddddd-agree",
+    ]  # fmt: skip
+    assert sum(float(t.budget.usd) for t in run.tasks) <= 0.05  # scaled down to the ceiling
+    assert all(float(t.budget.usd) == 0 for t in run.tasks if t.name.endswith(("-agree", "-facts")))
+    excerpt = manifests.tasks["verify-a-h-aaaaaaaaaaaa"]
+    env = {e.name: e.value for e in excerpt.env}
+    assert env["OUSAST_WORKSPACE_DIR"].endswith("/repo") and "/inputs/" in env["OUSAST_INPUT_CANDIDATES"]
+    workspace = manifests.workspaces["h-aaaaaaaaaaaa-side"]
+    paths = {f.path for f in workspace.files}
+    assert paths == {"repo/app.py", "inputs/candidates.json"}  # candidates outside the hunt's root
+    assert "cwe" not in yaml.safe_dump([workspace.metadata.name, *paths])  # the pair's name never reaches the sandbox
+    units = json.loads(files["units.json"])
+    assert "skipped" in units["h-cccccccccccc"] and "skipped" not in units["h-aaaaaaaaaaaa"]
+    agree = {e.name: e.value for e in manifests.tasks["agree-h-dddddddddddd"].env}
+    assert "OUSAST_INPUT_SITES" not in agree  # no declared sites: no site_match is computed
+
+
+def test_roles_run_groups_excerpts_and_pins(tmp_path: Path) -> None:
+    _, root = _render(tmp_path)
+    run, manifests = reconciler.load_run(root / "runs" / "hv-roles.yaml")
+    assert len(run.tasks) == 2 and sum(float(t.budget.usd) for t in run.tasks) <= 10.0
+    grouped = next(w for w in manifests.workspaces.values() if w.metadata.name.startswith("hv-roles"))
+    listing = json.loads(next(f.content for f in grouped.files if f.path == "inputs/files.json"))
+    assert listing == ["h-aaaaaaaaaaaa/app.py", "h-bbbbbbbbbbbb/f.c"]  # the oversized excerpt is left out
+    assert sum(len(f.content) for f in grouped.files) <= MAX_INLINE_BYTES + 200
+    pin = manifests.tasks["roles-h-dddddddddddd"]
+    assert {e.name: e.value for e in pin.env}["OUSAST_INPUT_FILES"].endswith("/files.json")
+
+
+def test_blob_sha1_is_gits_and_declaration_lines() -> None:
+    text = "def a():\n    pass\n\ndef b():\n    pass\n"
+    done = subprocess.run(["git", "hash-object", "--stdin"], input=text, capture_output=True, text=True, check=True)
+    assert blob_sha1(text) == done.stdout.strip()
+    lines = text.splitlines()
+    assert declaration_line(lines, "m.py", "b") == 4
+    assert declaration_line(lines, "m.py", "missing") == 1 and declaration_line(None, "m.py", "a") == 1