marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-17 12:09:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

bench(learn): harvest signals for the decision engine (decision-engine task 5)

Browse Source 
`ousast plane harvest` (plane/harvest.py) writes two Runs over a label snapshot:
verify a/b + agree on the verify-family units (both sides of each labelled pair as
an opaque files-only Workspace, candidates outside the hunt's root, no declared
sites; v1 and dev-php pins with repo-facts first) and model roles on the candidate
files (pair excerpts grouped, one Task per population pin). Budgets sum under the
ceiling by construction. New roles task template; the generator re-pins it.

Spent by attribution: verify $2.54 (1961 calls, 150 units, 182 hunts per pass),
roles $1.40 (1034 calls, 799 files, 0 unclassified chunks); DeepSeek balance
19.63 -> 18.23. Engine on the pair corpus (benchmarks/learn/engine_pairs.py,
frozen e6ed785, one 3g container) is PARTIAL: 384 of 686 sides at commit, still
running; unfinished sides stay missing. benchmarks/learn/build_features.py built
1712 static+plane records into the memory store; counts-only record under
benchmarks/measurements/2026-10-01-decision-engine-harvest/.

source/roles parts are `none`, not `failed`, for a language without a declaration
pattern (C). ops/ax/README: Substrate golden-actor disk leak and the inline
Workspace limit (~20 KB).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-01 14:04:28 +02:00
co-authored by Claude Opus 5.5
parent e6ed7854bb
commit 34da916030
21 changed files with 1739 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/measurements/2026-09-08-module-audit.json
+26 -5
View File
@@ -1,7 +1,7 @@
{
"modules": 143,
"modules": 144,
"counts": {
"load_bearing": 136,
"load_bearing": 137,
"standalone_capability": 7
},
"orphaned": [],
@@ -354,6 +354,7 @@
"cli",
"learn.features",
"learn.roles",
"plane.harvest",
"plane.tasks.features",
"plane.tasks.remember",
"plane.tasks.roles"
@@ -372,13 +373,14 @@
"classification": "load_bearing",
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
"importers": [
"cli"
"cli",
"plane.harvest"
]
},
{
"module": "learn.roles",
"classification": "load_bearing",
"reason": "imported by learn.features, plane.tasks.features, plane.tasks.roles",
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
"importers": [
"learn.features",
"plane.tasks.features",
@@ -709,6 +711,7 @@
"learn.labels",
"model.candidates",
"pair_gate",
"plane.harvest",
"plane.tasks.loop"
]
},
@@ -722,6 +725,7 @@
"plane.egress",
"plane.engine_alerts",
"plane.generate",
"plane.harvest",
"plane.memory",
"plane.reconciler",
"plane.router",
@@ -769,6 +773,15 @@
"module": "plane.generate",
"classification": "load_bearing",
"reason": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set",
"importers": [
"cli",
"plane.harvest"
]
},
{
"module": "plane.harvest",
"classification": "load_bearing",
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
"importers": [
"cli"
]
@@ -780,6 +793,7 @@
"importers": [
"plane.egress",
"plane.generate",
"plane.harvest",
"plane.reconciler",
"plane.router",
"plane.runner",
@@ -803,6 +817,7 @@
"reason": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`",
"importers": [
"plane.generate",
"plane.harvest",
"plane.memory",
"plane.workspaces"
]
@@ -833,6 +848,7 @@
"learn.roles",
"plane.engine_alerts",
"plane.generate",
"plane.harvest",
"plane.memory",
"plane.tasks.agree",
"plane.tasks.alerts",
@@ -888,6 +904,7 @@
"learn.labels",
"learn.roles",
"plane.generate",
"plane.harvest",
"plane.tasks.alerts",
"plane.tasks.features",
"plane.tasks.remember",
@@ -898,7 +915,9 @@
"module": "plane.tasks.roles",
"classification": "load_bearing",
"reason": "learned-decision-engine Req 8.1: model role classification per file chunk (model_sinks.py moved in); its Run template comes with the harvest (task 5)",
"importers": []
"importers": [
"plane.harvest"
]
},
{
"module": "plane.tasks.verify",
@@ -906,6 +925,7 @@
"reason": "ai-service-plane Req 5.3/2.1-2.3/7.2: the batched hunt with known callers, metered and resumable per file hunt",
"importers": [
"plane.generate",
"plane.harvest",
"plane.tasks.agree",
"plane.tasks.roles"
]
@@ -966,6 +986,7 @@
"model.partitions",
"model.scan",
"pairs",
"plane.harvest",
"plane.tasks.alerts",
"plane.tasks.features",
"plane.tasks.repo_facts",
benchmarks/measurements/2026-10-01-decision-engine-harvest/record.json
+477
View File
@@ -0,0 +1,477 @@
{
"balance_usd": {
"after_roles": 18.23,
"after_verify": 18.75,
"before": 19.63
},
"ceilings_usd": {
"roles": 10.0,
"verify": 10.0
},
"created": "2026-10-01",
"engine": {
"frozen_source_commit": "e6ed7854bb8ef17d6a4c39971ea253b4e385af23",
"image": "openultrasast:dev sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"pair_sides_total": 686,
"population_engine": "recorded protocol scans of independent-v1/v2 (vulnerable and fixed pins); dev-php not run (missing)",
"sides_missing": 302,
"sides_recorded": {
"failed (read its input, asked no question)": 286,
"none": 6,
"ran": 92
},
"status": "running in the background at commit time; unfinished sides are `missing` in the records, never zero"
},
"features": {
"instrument_states_plane": {
"access_control:agree": {
"none": 26
},
"access_control:delta": {
"not_applicable": 26
},
"access_control:engine": {
"missing": 6,
"ran": 20
},
"access_control:entry_points": {
"not_applicable": 26
},
"access_control:facts": {
"not_applicable": 26
},
"access_control:language": {
"ran": 26
},
"access_control:model_sinks": {
"ran": 26
},
"access_control:quick": {
"ran": 26
},
"access_control:roles": {
"failed": 13,
"ran": 13
},
"access_control:source": {
"failed": 13,
"ran": 13
},
"access_control:verify": {
"none": 26
},
"config_secrets:agree": {
"none": 4,
"ran": 31
},
"config_secrets:delta": {
"not_applicable": 35
},
"config_secrets:engine": {
"missing": 4,
"none": 23,
"ran": 8
},
"config_secrets:entry_points": {
"not_applicable": 8,
"ran": 27
},
"config_secrets:facts": {
"not_applicable": 8,
"ran": 27
},
"config_secrets:language": {
"ran": 35
},
"config_secrets:model_sinks": {
"ran": 35
},
"config_secrets:quick": {
"ran": 35
},
"config_secrets:roles": {
"failed": 14,
"ran": 21
},
"config_secrets:source": {
"failed": 4,
"ran": 31
},
"config_secrets:verify": {
"none": 4,
"ran": 31
},
"deserialization:agree": {
"none": 40
},
"deserialization:delta": {
"not_applicable": 40
},
"deserialization:engine": {
"failed": 21,
"ran": 19
},
"deserialization:entry_points": {
"not_applicable": 40
},
"deserialization:facts": {
"not_applicable": 40
},
"deserialization:language": {
"ran": 40
},
"deserialization:model_sinks": {
"ran": 40
},
"deserialization:quick": {
"ran": 40
},
"deserialization:roles": {
"failed": 20,
"ran": 20
},
"deserialization:source": {
"failed": 20,
"ran": 20
},
"deserialization:verify": {
"none": 40
},
"injection:agree": {
"missing": 4,
"none": 25,
"ran": 139
},
"injection:delta": {
"not_applicable": 168
},
"injection:engine": {
"failed": 22,
"missing": 78,
"none": 16,
"ran": 52
},
"injection:entry_points": {
"not_applicable": 70,
"ran": 98
},
"injection:facts": {
"not_applicable": 70,
"ran": 98
},
"injection:language": {
"ran": 168
},
"injection:model_sinks": {
"missing": 4,
"ran": 164
},
"injection:quick": {
"none": 2,
"ran": 166
},
"injection:roles": {
"failed": 44,
"none": 2,
"ran": 122
},
"injection:source": {
"failed": 36,
"none": 2,
"ran": 130
},
"injection:verify": {
"missing": 4,
"none": 25,
"ran": 139
},
"memory:agree": {
"none": 346
},
"memory:delta": {
"not_applicable": 346
},
"memory:engine": {
"failed": 116,
"missing": 218,
"none": 6,
"ran": 6
},
"memory:entry_points": {
"not_applicable": 346
},
"memory:facts": {
"not_applicable": 346
},
"memory:language": {
"ran": 346
},
"memory:model_sinks": {
"missing": 4,
"ran": 342
},
"memory:quick": {
"none": 4,
"ran": 342
},
"memory:roles": {
"none": 346
},
"memory:source": {
"none": 346
},
"memory:verify": {
"none": 346
},
"output_encoding:agree": {
"none": 112
},
"output_encoding:delta": {
"not_applicable": 112
},
"output_encoding:engine": {
"failed": 24,
"missing": 82,
"ran": 6
},
"output_encoding:entry_points": {
"not_applicable": 112
},
"output_encoding:facts": {
"not_applicable": 112
},
"output_encoding:language": {
"ran": 112
},
"output_encoding:model_sinks": {
"ran": 112
},
"output_encoding:quick": {
"ran": 112
},
"output_encoding:roles": {
"failed": 16,
"ran": 96
},
"output_encoding:source": {
"failed": 16,
"ran": 96
},
"output_encoding:verify": {
"none": 112
},
"path:agree": {
"none": 29
},
"path:delta": {
"not_applicable": 29
},
"path:engine": {
"failed": 22,
"missing": 5,
"ran": 2
},
"path:entry_points": {
"not_applicable": 28,
"ran": 1
},
"path:facts": {
"not_applicable": 28,
"ran": 1
},
"path:language": {
"ran": 29
},
"path:model_sinks": {
"ran": 29
},
"path:quick": {
"ran": 29
},
"path:roles": {
"failed": 15,
"ran": 14
},
"path:source": {
"failed": 15,
"ran": 14
},
"path:verify": {
"none": 29
},
"unknown:agree": {
"none": 4
},
"unknown:delta": {
"not_applicable": 4
},
"unknown:engine": {
"missing": 2,
"none": 2
},
"unknown:entry_points": {
"not_applicable": 4
},
"unknown:facts": {
"not_applicable": 4
},
"unknown:language": {
"ran": 4
},
"unknown:model_sinks": {
"ran": 4
},
"unknown:quick": {
"ran": 4
},
"unknown:roles": {
"none": 4
},
"unknown:source": {
"none": 4
},
"unknown:verify": {
"none": 4
},
"untrusted_destination:agree": {
"none": 11,
"ran": 85
},
"untrusted_destination:delta": {
"not_applicable": 96
},
"untrusted_destination:engine": {
"failed": 38,
"missing": 12,
"none": 17,
"ran": 29
},
"untrusted_destination:entry_points": {
"not_applicable": 52,
"ran": 44
},
"untrusted_destination:facts": {
"not_applicable": 52,
"ran": 44
},
"untrusted_destination:language": {
"ran": 96
},
"untrusted_destination:model_sinks": {
"ran": 96
},
"untrusted_destination:quick": {
"ran": 96
},
"untrusted_destination:roles": {
"failed": 39,
"ran": 57
},
"untrusted_destination:source": {
"failed": 33,
"ran": 63
},
"untrusted_destination:verify": {
"none": 11,
"ran": 85
}
},
"records": {
"plane:access_control": 26,
"plane:config_secrets": 35,
"plane:deserialization": 40,
"plane:injection": 168,
"plane:memory": 346,
"plane:output_encoding": 112,
"plane:path": 29,
"plane:unknown": 4,
"plane:untrusted_destination": 96,
"static:access_control": 26,
"static:config_secrets": 35,
"static:deserialization": 40,
"static:injection": 168,
"static:memory": 346,
"static:output_encoding": 112,
"static:path": 29,
"static:unknown": 4,
"static:untrusted_destination": 96
},
"skipped_label_rows": {},
"store": "file://~/ousast-results/plane/memory (kind features, run harvest)"
},
"host_peaks": {
"disk_free_mb_min": 960,
"disk_incident": "Substrate kept every golden actor (~24 MB each, deletion failing on a missing snapshot bucket); free disk fell to 0.95 GB, runs were stopped, the bucket created and orphaned actor directories removed",
"mem_available_mb_min": 1990
},
"label_snapshot_sha256": "18337014d8a5c8df67e06f4e9d8e72c1fb8f82b1ed1b9a18836ab37e6cf4fdc9",
"note": "Harvest of the learned decision engine (task 5), COUNTS ONLY. Spend is the plane's attribution table (`ousast plane status`, task meters priced at plane/models/deepseek-flash.yaml); the DeepSeek account moved less (balance below). Feature records are in the local memory store (kind `features`, run `harvest`); the label rows, the units index and the manifests name functions and stay local.",
"roles": {
"attribution": {
"cache_hit_tokens": null,
"calls": 1034,
"output_tokens": null,
"prompt_tokens": null,
"usd": 1.4036
},
"cache_hit_tokens": 329600,
"chunks": 1034,
"files": 799,
"output_tokens": 370155,
"prompt_tokens": 2398669,
"task_budget_sum_usd": 4.917,
"tasks_done": 151,
"unclassified_chunks": 0
},
"runner_image": "localhost:5001/ousast-runner@sha256:412db0a1504af8f1b23a395af8d1dcbd152c1bb6d16594f2b733067163a31884",
"verify": {
"attribution": {
"cache_hit_tokens": 6275060,
"calls": 1961,
"output_tokens": 306941,
"prompt_tokens": 10921167,
"usd": 2.5373
},
"file_hunts_done": 364,
"flagged_by_pass_units": {
"dev-php:vulnerable": {
"both": 2
},
"pairs:fixed": {
"both": 4,
"none": 58,
"one": 1
},
"pairs:vulnerable": {
"both": 45,
"none": 11,
"one": 7
},
"population-v1:fixed": {
"both": 5,
"none": 2,
"one": 4
},
"population-v1:vulnerable": {
"both": 8,
"none": 2,
"one": 1
}
},
"hunts_per_pass": 182,
"infrastructure_failures": "ax apply / actor template not found for Workspaces over the inline limit (units then skipped); two tasks interrupted by a disk stop were re-run",
"skipped_units": {
"pairs": 4,
"reason": "excerpt over the 20 KB inline Workspace limit"
},
"task_budget_sum_usd": 8.624,
"tasks_done": 474,
"units": {
"dev-php:vulnerable": 2,
"pairs:fixed": 63,
"pairs:vulnerable": 63,
"population-v1:fixed": 11,
"population-v1:vulnerable": 11
}
}
}