mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5.5
b68fe27c74
`ousast plane workspaces --validation-set ... --loop` appends the loop to a Run:
per case `alerts` (plane/tasks/alerts.py: the quick-mode scan on the vulnerable
and the fixed pin, a fixed-pin Workspace per case, alerts attributed to their
enclosing function by the repo-facts patterns, `in_fix_range` from the fix's
old-side and new-side ranges, failing on an unread tree) feeding `remember`,
then `memory-snapshot` -> `loop-measure` -> `loop-propose` -> `loop-improve`
(plane/tasks/loop.py). Every new task binds no Model and has budget
{usd: 0, calls: 0}. The snapshot is the store's rows after the train-on-test
guard, written on the host by memory.seed (a sandboxed run of it fails loudly);
propose runs M1/M2 against this repository at the commit under test; improve
runs one unchanged evolve.run_round and delivers gate.json, journal.json, the
sidecar, rule_policy.json only when accepted, and proposal_outcome rows that
the host ingests. Adoption stays a maintainer commit.
`remember` is now a Run entry per case (the image contains it), with the facts
row keyed by the image the facts were computed on (repo-facts records it in
its summary; unknown -> the row is omitted, never guessed). Alert rows carry
their own pin; improve/memory.py counts a repeated alert once.
Runner image rebuilt (+0.39 MB: only the new modules; the quick scan needs no
extra and the rulesets were already in the package), pushed, plane/ re-pinned
and regenerated (validation-46: 105 entries). Fixture test runs the whole loop
graph through the fake ax, whose new `exec` mode runs the real task modules.
Live: one loop Run over three validation-46 cases on the local ax, recorded in
benchmarks/measurements/2026-09-30-harnessx-removal-loop/ (all done, $0 new
spend, 0 proposals, gate verdict no_proposals).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
141 lines
2.9 KiB
JSON
141 lines
2.9 KiB
JSON
{
|
|
"inputs_bytes": {
|
|
"froxlor_admin_ip_second_order_sqli": 5943,
|
|
"lmdeploy_media_url_ssrf": 43730,
|
|
"wp_gopay_log_filter_sqli": 1947
|
|
},
|
|
"rows": {
|
|
"run": 87,
|
|
"snapshot": 119,
|
|
"union": 191
|
|
},
|
|
"run": "validation-46",
|
|
"run_metrics": {
|
|
"agreed": 4,
|
|
"alerts": {
|
|
"c-shell-exec": {
|
|
"fixed": 2,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 2
|
|
},
|
|
"c-unsafe-memory-copy": {
|
|
"fixed": 2,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 2
|
|
},
|
|
"python-shell-true": {
|
|
"fixed": 1,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 1
|
|
},
|
|
"python-unsafe-deserialization": {
|
|
"fixed": 8,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 8
|
|
},
|
|
"python-unsafe-eval": {
|
|
"fixed": 23,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 23
|
|
}
|
|
},
|
|
"candidates": 5,
|
|
"cases": 3,
|
|
"declared_sites_agreed": 4,
|
|
"declared_sites_matched": 5,
|
|
"dispute_rate": {
|
|
"injection": {
|
|
"candidates": 4,
|
|
"disputed": 1,
|
|
"rate": 0.25
|
|
},
|
|
"untrusted_destination": {
|
|
"candidates": 1,
|
|
"disputed": 0,
|
|
"rate": 0.0
|
|
}
|
|
},
|
|
"disputed": 1,
|
|
"rejected": 0,
|
|
"tiebreaks": 3,
|
|
"usd": 0.130134,
|
|
"usd_per_candidate": 0.026027
|
|
},
|
|
"store_metrics": {
|
|
"agreed": 28,
|
|
"alerts": {},
|
|
"candidates": 43,
|
|
"cases": 14,
|
|
"declared_sites_agreed": 16,
|
|
"declared_sites_matched": 19,
|
|
"dispute_rate": {
|
|
"injection": {
|
|
"candidates": 38,
|
|
"disputed": 5,
|
|
"rate": 0.1316
|
|
},
|
|
"untrusted_destination": {
|
|
"candidates": 5,
|
|
"disputed": 0,
|
|
"rate": 0.0
|
|
}
|
|
},
|
|
"disputed": 5,
|
|
"rejected": 10,
|
|
"tiebreaks": 10,
|
|
"usd": 0.856679,
|
|
"usd_per_candidate": 0.019923
|
|
},
|
|
"union_metrics": {
|
|
"agreed": 28,
|
|
"alerts": {
|
|
"c-shell-exec": {
|
|
"fixed": 2,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 2
|
|
},
|
|
"c-unsafe-memory-copy": {
|
|
"fixed": 2,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 2
|
|
},
|
|
"python-shell-true": {
|
|
"fixed": 1,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 1
|
|
},
|
|
"python-unsafe-deserialization": {
|
|
"fixed": 8,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 8
|
|
},
|
|
"python-unsafe-eval": {
|
|
"fixed": 23,
|
|
"fixed_in_fix_range": 0,
|
|
"vulnerable": 23
|
|
}
|
|
},
|
|
"candidates": 43,
|
|
"cases": 14,
|
|
"declared_sites_agreed": 16,
|
|
"declared_sites_matched": 19,
|
|
"dispute_rate": {
|
|
"injection": {
|
|
"candidates": 38,
|
|
"disputed": 5,
|
|
"rate": 0.1316
|
|
},
|
|
"untrusted_destination": {
|
|
"candidates": 5,
|
|
"disputed": 0,
|
|
"rate": 0.0
|
|
}
|
|
},
|
|
"disputed": 5,
|
|
"rejected": 10,
|
|
"tiebreaks": 10,
|
|
"usd": 0.856679,
|
|
"usd_per_candidate": 0.019923
|
|
}
|
|
}
|