mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
Marius CiepluchandClaude Opus 5.5
b68fe27c74
`ousast plane workspaces --validation-set ... --loop` appends the loop to a Run:
per case `alerts` (plane/tasks/alerts.py: the quick-mode scan on the vulnerable
and the fixed pin, a fixed-pin Workspace per case, alerts attributed to their
enclosing function by the repo-facts patterns, `in_fix_range` from the fix's
old-side and new-side ranges, failing on an unread tree) feeding `remember`,
then `memory-snapshot` -> `loop-measure` -> `loop-propose` -> `loop-improve`
(plane/tasks/loop.py). Every new task binds no Model and has budget
{usd: 0, calls: 0}. The snapshot is the store's rows after the train-on-test
guard, written on the host by memory.seed (a sandboxed run of it fails loudly);
propose runs M1/M2 against this repository at the commit under test; improve
runs one unchanged evolve.run_round and delivers gate.json, journal.json, the
sidecar, rule_policy.json only when accepted, and proposal_outcome rows that
the host ingests. Adoption stays a maintainer commit.
`remember` is now a Run entry per case (the image contains it), with the facts
row keyed by the image the facts were computed on (repo-facts records it in
its summary; unknown -> the row is omitted, never guessed). Alert rows carry
their own pin; improve/memory.py counts a repeated alert once.
Runner image rebuilt (+0.39 MB: only the new modules; the quick scan needs no
extra and the rulesets were already in the package), pushed, plane/ re-pinned
and regenerated (validation-46: 105 entries). Fixture test runs the whole loop
graph through the fake ax, whose new `exec` mode runs the real task modules.
Live: one loop Run over three validation-46 cases on the local ax, recorded in
benchmarks/measurements/2026-09-30-harnessx-removal-loop/ (all done, $0 new
spend, 0 proposals, gate verdict no_proposals).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
321 lines
9.8 KiB
YAML
321 lines
9.8 KiB
YAML
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set /tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr4/set/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --loop --loop-commit 1ad472c68d759e4ef010bd7a2e23532b1de538a9 --loop-repo https://github.com/norandom/OpenUltraSAST.git --loop-manifest benchmarks/manifests/java-spring-boot-vulnerable.toml`; do not edit, regenerate.
|
|
# 3 cases, 5 candidates (5 kept by the recorded triage); usd ceiling 0.33.
|
|
apiVersion: openultrasast.io/v1alpha1
|
|
kind: Run
|
|
metadata:
|
|
name: validation-46
|
|
annotations:
|
|
openultrasast.io/population: population-v2
|
|
openultrasast.io/split: validation
|
|
spec:
|
|
tasks:
|
|
- name: froxlor-admin-ip-second-order-sqli-facts
|
|
task: repo-facts-froxlor-admin-ip-second-order-sqli
|
|
outputs:
|
|
- facts.json
|
|
budget: &id001
|
|
usd: 0
|
|
calls: 0
|
|
serialize: repo-facts
|
|
- name: froxlor-admin-ip-second-order-sqli-va
|
|
task: verify-a-froxlor-admin-ip-second-order-sqli
|
|
inputs:
|
|
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.07
|
|
calls: 16
|
|
- name: froxlor-admin-ip-second-order-sqli-vb
|
|
task: verify-b-froxlor-admin-ip-second-order-sqli
|
|
inputs:
|
|
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.07
|
|
calls: 16
|
|
- name: froxlor-admin-ip-second-order-sqli-agree
|
|
task: agree-froxlor-admin-ip-second-order-sqli
|
|
inputs:
|
|
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
|
|
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
|
|
outputs:
|
|
- agreed.json
|
|
- disputed.json
|
|
budget: *id001
|
|
- name: lmdeploy-media-url-ssrf-facts
|
|
task: repo-facts-lmdeploy-media-url-ssrf
|
|
outputs:
|
|
- facts.json
|
|
budget: &id002
|
|
usd: 0
|
|
calls: 0
|
|
serialize: repo-facts
|
|
- name: lmdeploy-media-url-ssrf-va
|
|
task: verify-a-lmdeploy-media-url-ssrf
|
|
inputs:
|
|
facts: lmdeploy-media-url-ssrf-facts/facts.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.02
|
|
calls: 8
|
|
- name: lmdeploy-media-url-ssrf-vb
|
|
task: verify-b-lmdeploy-media-url-ssrf
|
|
inputs:
|
|
facts: lmdeploy-media-url-ssrf-facts/facts.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.02
|
|
calls: 8
|
|
- name: lmdeploy-media-url-ssrf-agree
|
|
task: agree-lmdeploy-media-url-ssrf
|
|
inputs:
|
|
pass_a: lmdeploy-media-url-ssrf-va/units.jsonl
|
|
pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl
|
|
outputs:
|
|
- agreed.json
|
|
- disputed.json
|
|
budget: *id002
|
|
- name: wp-gopay-log-filter-sqli-facts
|
|
task: repo-facts-wp-gopay-log-filter-sqli
|
|
outputs:
|
|
- facts.json
|
|
budget: &id003
|
|
usd: 0
|
|
calls: 0
|
|
serialize: repo-facts
|
|
- name: wp-gopay-log-filter-sqli-va
|
|
task: verify-a-wp-gopay-log-filter-sqli
|
|
inputs:
|
|
facts: wp-gopay-log-filter-sqli-facts/facts.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.02
|
|
calls: 8
|
|
- name: wp-gopay-log-filter-sqli-vb
|
|
task: verify-b-wp-gopay-log-filter-sqli
|
|
inputs:
|
|
facts: wp-gopay-log-filter-sqli-facts/facts.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.02
|
|
calls: 8
|
|
- name: wp-gopay-log-filter-sqli-agree
|
|
task: agree-wp-gopay-log-filter-sqli
|
|
inputs:
|
|
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
|
|
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
|
|
outputs:
|
|
- agreed.json
|
|
- disputed.json
|
|
budget: *id003
|
|
- name: froxlor-admin-ip-second-order-sqli-vc
|
|
task: verify-c-froxlor-admin-ip-second-order-sqli
|
|
dependsOn:
|
|
- froxlor-admin-ip-second-order-sqli-agree
|
|
inputs:
|
|
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
|
|
only: froxlor-admin-ip-second-order-sqli-agree/disputed.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.07
|
|
calls: 16
|
|
- name: froxlor-admin-ip-second-order-sqli-final
|
|
task: agree-froxlor-admin-ip-second-order-sqli
|
|
inputs:
|
|
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
|
|
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
|
|
pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl
|
|
outputs:
|
|
- agreed.json
|
|
- disputed.json
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: lmdeploy-media-url-ssrf-vc
|
|
task: verify-c-lmdeploy-media-url-ssrf
|
|
dependsOn:
|
|
- lmdeploy-media-url-ssrf-agree
|
|
inputs:
|
|
facts: lmdeploy-media-url-ssrf-facts/facts.json
|
|
only: lmdeploy-media-url-ssrf-agree/disputed.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.02
|
|
calls: 8
|
|
- name: lmdeploy-media-url-ssrf-final
|
|
task: agree-lmdeploy-media-url-ssrf
|
|
inputs:
|
|
pass_a: lmdeploy-media-url-ssrf-va/units.jsonl
|
|
pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl
|
|
pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl
|
|
outputs:
|
|
- agreed.json
|
|
- disputed.json
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: wp-gopay-log-filter-sqli-vc
|
|
task: verify-c-wp-gopay-log-filter-sqli
|
|
dependsOn:
|
|
- wp-gopay-log-filter-sqli-agree
|
|
inputs:
|
|
facts: wp-gopay-log-filter-sqli-facts/facts.json
|
|
only: wp-gopay-log-filter-sqli-agree/disputed.json
|
|
outputs:
|
|
- units.jsonl
|
|
- summary.json
|
|
budget:
|
|
usd: 0.02
|
|
calls: 8
|
|
- name: wp-gopay-log-filter-sqli-final
|
|
task: agree-wp-gopay-log-filter-sqli
|
|
inputs:
|
|
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
|
|
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
|
|
pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl
|
|
outputs:
|
|
- agreed.json
|
|
- disputed.json
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: froxlor-admin-ip-second-order-sqli-alerts
|
|
task: alerts-froxlor-admin-ip-second-order-sqli
|
|
outputs:
|
|
- alerts.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
serialize: engine
|
|
- name: froxlor-admin-ip-second-order-sqli-remember
|
|
task: remember-froxlor-admin-ip-second-order-sqli
|
|
inputs:
|
|
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
|
|
facts_summary: froxlor-admin-ip-second-order-sqli-facts/summary.json
|
|
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
|
|
summary_a: froxlor-admin-ip-second-order-sqli-va/summary.json
|
|
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
|
|
summary_b: froxlor-admin-ip-second-order-sqli-vb/summary.json
|
|
pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl
|
|
summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json
|
|
agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json
|
|
alerts: froxlor-admin-ip-second-order-sqli-alerts/alerts.jsonl
|
|
outputs:
|
|
- memory.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: lmdeploy-media-url-ssrf-alerts
|
|
task: alerts-lmdeploy-media-url-ssrf
|
|
outputs:
|
|
- alerts.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
serialize: engine
|
|
- name: lmdeploy-media-url-ssrf-remember
|
|
task: remember-lmdeploy-media-url-ssrf
|
|
inputs:
|
|
facts: lmdeploy-media-url-ssrf-facts/facts.json
|
|
facts_summary: lmdeploy-media-url-ssrf-facts/summary.json
|
|
pass_a: lmdeploy-media-url-ssrf-va/units.jsonl
|
|
summary_a: lmdeploy-media-url-ssrf-va/summary.json
|
|
pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl
|
|
summary_b: lmdeploy-media-url-ssrf-vb/summary.json
|
|
pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl
|
|
summary_c: lmdeploy-media-url-ssrf-vc/summary.json
|
|
agreed: lmdeploy-media-url-ssrf-final/agreed.json
|
|
alerts: lmdeploy-media-url-ssrf-alerts/alerts.jsonl
|
|
outputs:
|
|
- memory.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: wp-gopay-log-filter-sqli-alerts
|
|
task: alerts-wp-gopay-log-filter-sqli
|
|
outputs:
|
|
- alerts.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
serialize: engine
|
|
- name: wp-gopay-log-filter-sqli-remember
|
|
task: remember-wp-gopay-log-filter-sqli
|
|
inputs:
|
|
facts: wp-gopay-log-filter-sqli-facts/facts.json
|
|
facts_summary: wp-gopay-log-filter-sqli-facts/summary.json
|
|
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
|
|
summary_a: wp-gopay-log-filter-sqli-va/summary.json
|
|
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
|
|
summary_b: wp-gopay-log-filter-sqli-vb/summary.json
|
|
pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl
|
|
summary_c: wp-gopay-log-filter-sqli-vc/summary.json
|
|
agreed: wp-gopay-log-filter-sqli-final/agreed.json
|
|
alerts: wp-gopay-log-filter-sqli-alerts/alerts.jsonl
|
|
outputs:
|
|
- memory.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: memory-snapshot
|
|
task: memory-snapshot
|
|
outputs:
|
|
- rows.jsonl
|
|
- index.json
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: loop-measure
|
|
task: loop-measure
|
|
inputs:
|
|
remember_froxlor-admin-ip-second-order-sqli: froxlor-admin-ip-second-order-sqli-remember/memory.jsonl
|
|
remember_lmdeploy-media-url-ssrf: lmdeploy-media-url-ssrf-remember/memory.jsonl
|
|
remember_wp-gopay-log-filter-sqli: wp-gopay-log-filter-sqli-remember/memory.jsonl
|
|
snapshot: memory-snapshot/rows.jsonl
|
|
outputs:
|
|
- measure.json
|
|
- rows.jsonl
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: loop-propose
|
|
task: loop-propose
|
|
inputs:
|
|
measure: loop-measure/measure.json
|
|
rows: loop-measure/rows.jsonl
|
|
index: memory-snapshot/index.json
|
|
outputs:
|
|
- proposals.jsonl
|
|
- memory_proposals.jsonl
|
|
- signals.json
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
- name: loop-improve
|
|
task: loop-improve
|
|
inputs:
|
|
proposals: loop-propose/proposals.jsonl
|
|
outputs:
|
|
- gate.json
|
|
- journal.json
|
|
budget:
|
|
usd: 0
|
|
calls: 0
|
|
serialize: engine
|