marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

bench(plane): correct the increment reading -- 17/20 flagged in either pass, not 19

Browse Source 
agree's declared_sites_matched counts candidate coverage, not detection. Recounted
from agreed.json rows, plane passes flagged 17 of 20 declared sites against the
reference's 19 (budibase read and fuxa getDaqValue missed by both passes). The
earlier "detection unchanged" reading was wrong.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 08:20:50 +02:00
co-authored by Claude Opus 5.5
parent c5e6ed3221
commit 2f1b852966
1 changed files with 77 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/independent/plane-increment-1.json
+77 -22
View File
@@ -1,37 +1,92 @@
{ {
"measurement": "ai-service-plane first increment (Requirement 6): repo-facts + verify (2 passes) + agree for the 46-candidate validation set, executed by the ax-backed reconciler on google/ax + Agent Substrate (kind, gVisor) on the 7 GB host", "measurement": "ai-service-plane first increment (Requirement 6): repo-facts + verify (2 passes) + agree for the 46-candidate validation set, executed by the ax-backed reconciler on google/ax + Agent Substrate (kind, gVisor) on the 7 GB host",
"date": "2026-09-29", "date": "2026-09-29",
"run": {"name": "validation-46", "manifest": "plane/runs/validation-46.yaml", "tasks": 60, "workers": 2, "status": "done", "run": {
"results": "~/ousast-results/plane/validation-46/", "model": "deepseek-flash", "runner_image": "localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"}, "name": "validation-46",
"manifest": "plane/runs/validation-46.yaml",
"tasks": 60,
"workers": 2,
"status": "done",
"results": "~/ousast-results/plane/validation-46/",
"model": "deepseek-flash",
"runner_image": "localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"
},
"reference": "benchmarks/independent/verifier-batched-check-2026-09-29.json (script pipeline, same cases, candidates, pins and recorded triage)", "reference": "benchmarks/independent/verifier-batched-check-2026-09-29.json (script pipeline, same cases, candidates, pins and recorded triage)",
"gate": { "gate": {
"declared_sites_agreed": {"plane": 14, "reference": 16, "of": 20, "required": ">= 16", "met": false}, "declared_sites_agreed": {
"cost_per_candidate_with_recorded_triage_usd": {"plane": 0.0194, "plane_over_46": 0.0181, "reference": 0.022, "required": "< 0.022", "met": true}, "plane": 14,
"reference": 16,
"of": 20,
"required": ">= 16",
"met": false
},
"cost_per_candidate_with_recorded_triage_usd": {
"plane": 0.0194,
"plane_over_46": 0.0181,
"reference": 0.022,
"required": "< 0.022",
"met": true
},
"result": "NOT MET (recall criterion)" "result": "NOT MET (recall criterion)"
}, },
"detail": { "detail": {
"declared_sites_found_in_either_pass": {"plane": 19, "reference": 19, "of": 20}, "agreed_candidates": 23,
"agreed_candidates": 23, "disputed_candidates": 10, "candidates_after_recorded_triage": 43, "disputed_candidates": 10,
"plane_usd": 0.7471, "recorded_triage_usd": 0.0872, "usd_with_triage": 0.8343, "reference_usd": 1.0123, "candidates_after_recorded_triage": 43,
"calls": 357, "prompt_tokens": 3820395, "cache_hit_tokens": 2385152, "output_tokens": 62257, "plane_usd": 0.7471,
"turns_per_hunt": 5.37, "hunts": 54, "recorded_triage_usd": 0.0872,
"usd_with_triage": 0.8343,
"reference_usd": 1.0123,
"calls": 357,
"prompt_tokens": 3820395,
"cache_hit_tokens": 2385152,
"output_tokens": 62257,
"turns_per_hunt": 5.37,
"hunts": 54,
"per_case_sites_agreed_vs_reference": { "per_case_sites_agreed_vs_reference": {
"budibase-mongo-template-nosqli": "1/2 vs 2/2 (one site found in one pass only)", "budibase-mongo-template-nosqli": "1/2 vs 2/2 (read: flagged by neither plane pass; the reference agreed on it)",
"froxlor-admin-ip-second-order-sqli": "0/3 vs 2/3 (two sites found in one pass only)", "froxlor-admin-ip-second-order-sqli": "0/3 vs 2/3 (all three flagged by pass b only)",
"unleash-addon-webhook-ssrf": "1/1 vs 0/1 (reference found it in one pass only)", "unleash-addon-webhook-ssrf": "1/1 vs 0/1 (reference found it in one pass only)",
"fuxa-tdengine-escape-sqli": "0/1 vs 0/1", "winml-cli-serve-cors": "0/1 vs 0/1 (triage drops the only candidate)", "fuxa-tdengine-escape-sqli": "0/1 vs 0/1 (getDaqValue: neither plane pass; the reference found it in one pass)",
"winml-cli-serve-cors": "0/1 vs 0/1 (triage drops the only candidate)",
"all_other_cases": "equal" "all_other_cases": "equal"
},
"declared_sites_flagged_in_either_pass": {
"plane": 17,
"reference": 19,
"of": 20,
"definition": "declared sites whose candidate pass a or pass b flagged; reference = 16 agreed + fuxa, froxlor validateIpAddresses, unleash found in one pass"
},
"declared_sites_covered_by_candidates": {
"plane": 19,
"of": 20,
"definition": "agree's metric declared_sites_matched: candidates whose site falls on a declared site, flagged or not (winml has no candidate)"
} }
}, },
"reading": "Detection is unchanged: both runs find 19 of 20 declared sites in at least one pass. The two-pass agreement rule's known flicker moved three sites into 'disputed' (net -2 against the reference). Cost fell 17% with the repo-facts callers line in the prompt and prompt caching (62% of prompt tokens were cache hits). By the pre-registered rule the increment is NOT MET and the design is revisited before task 9; the evidence points at the agreement rule (2-of-2 passes), not at the plane.", "reading": "Detection is lower, not only agreement: plane passes flagged 17 of 20 declared sites in at least one pass against the reference's 19 (budibase read and fuxa getDaqValue were flagged by neither plane pass), and two-pass agreement moved the three froxlor sites (pass b only) into 'disputed'; unleash moved the other way. Cost fell 17% (62% of prompt tokens were cache hits). By the pre-registered rule the increment is NOT MET and the design is revisited before task 9.",
"dev_token_report": {"transcript": "the whole development session (c09e30b2), not the increment alone", "unique_chars": "7.3M (~1.81M tokens)", "dev_token_report": {
"tool_use_input_pct": 45.4, "tool_result_pct": 40.1, "tool_io_pct": 85.5, "previous_tool_io_pct": 90, "transcript": "the whole development session (c09e30b2), not the increment alone",
"tool_calls": 2701}, "unique_chars": "7.3M (~1.81M tokens)",
"tool_use_input_pct": 45.4,
"tool_result_pct": 40.1,
"tool_io_pct": 85.5,
"previous_tool_io_pct": 90,
"tool_calls": 2701
},
"live_defects_found_by_running_on_ax": [ "live_defects_found_by_running_on_ax": [
"no Substrate WorkerPool", "tag-pinned images rejected", "ax CLI/server skew", "unknown-field rejection (atespace, commit)", "no Substrate WorkerPool",
"server-set fields in AX_TASK_YAML", "tasks created Suspended; no Completed phase", "runner exited after the command", "tag-pinned images rejected",
"golden-snapshot boot ran the command", "egress gateway not deployed; deny-by-default per-actor policy", "ax CLI/server skew",
"gateway dials the address the actor dialled", "workspace clone before the actor was running", "unknown-field rejection (atespace, commit)",
"32 KB env limit on inlined inputs", "63-byte template name limit" "server-set fields in AX_TASK_YAML",
] "tasks created Suspended; no Completed phase",
"runner exited after the command",
"golden-snapshot boot ran the command",
"egress gateway not deployed; deny-by-default per-actor policy",
"gateway dials the address the actor dialled",
"workspace clone before the actor was running",
"32 KB env limit on inlined inputs",
"63-byte template name limit"
],
"correction": "2026-09-30: the first version of this record read agree's declared_sites_matched (coverage by candidates, 19) as 'found in either pass' and concluded detection was unchanged; recounted from agreed.json rows it is 17 vs the reference's 19."
} }