mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
2f1b852966
agree's declared_sites_matched counts candidate coverage, not detection. Recounted from agreed.json rows, plane passes flagged 17 of 20 declared sites against the reference's 19 (budibase read and fuxa getDaqValue missed by both passes). The earlier "detection unchanged" reading was wrong. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
93 lines
4.2 KiB
JSON
93 lines
4.2 KiB
JSON
{
|
|
"measurement": "ai-service-plane first increment (Requirement 6): repo-facts + verify (2 passes) + agree for the 46-candidate validation set, executed by the ax-backed reconciler on google/ax + Agent Substrate (kind, gVisor) on the 7 GB host",
|
|
"date": "2026-09-29",
|
|
"run": {
|
|
"name": "validation-46",
|
|
"manifest": "plane/runs/validation-46.yaml",
|
|
"tasks": 60,
|
|
"workers": 2,
|
|
"status": "done",
|
|
"results": "~/ousast-results/plane/validation-46/",
|
|
"model": "deepseek-flash",
|
|
"runner_image": "localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"
|
|
},
|
|
"reference": "benchmarks/independent/verifier-batched-check-2026-09-29.json (script pipeline, same cases, candidates, pins and recorded triage)",
|
|
"gate": {
|
|
"declared_sites_agreed": {
|
|
"plane": 14,
|
|
"reference": 16,
|
|
"of": 20,
|
|
"required": ">= 16",
|
|
"met": false
|
|
},
|
|
"cost_per_candidate_with_recorded_triage_usd": {
|
|
"plane": 0.0194,
|
|
"plane_over_46": 0.0181,
|
|
"reference": 0.022,
|
|
"required": "< 0.022",
|
|
"met": true
|
|
},
|
|
"result": "NOT MET (recall criterion)"
|
|
},
|
|
"detail": {
|
|
"agreed_candidates": 23,
|
|
"disputed_candidates": 10,
|
|
"candidates_after_recorded_triage": 43,
|
|
"plane_usd": 0.7471,
|
|
"recorded_triage_usd": 0.0872,
|
|
"usd_with_triage": 0.8343,
|
|
"reference_usd": 1.0123,
|
|
"calls": 357,
|
|
"prompt_tokens": 3820395,
|
|
"cache_hit_tokens": 2385152,
|
|
"output_tokens": 62257,
|
|
"turns_per_hunt": 5.37,
|
|
"hunts": 54,
|
|
"per_case_sites_agreed_vs_reference": {
|
|
"budibase-mongo-template-nosqli": "1/2 vs 2/2 (read: flagged by neither plane pass; the reference agreed on it)",
|
|
"froxlor-admin-ip-second-order-sqli": "0/3 vs 2/3 (all three flagged by pass b only)",
|
|
"unleash-addon-webhook-ssrf": "1/1 vs 0/1 (reference found it in one pass only)",
|
|
"fuxa-tdengine-escape-sqli": "0/1 vs 0/1 (getDaqValue: neither plane pass; the reference found it in one pass)",
|
|
"winml-cli-serve-cors": "0/1 vs 0/1 (triage drops the only candidate)",
|
|
"all_other_cases": "equal"
|
|
},
|
|
"declared_sites_flagged_in_either_pass": {
|
|
"plane": 17,
|
|
"reference": 19,
|
|
"of": 20,
|
|
"definition": "declared sites whose candidate pass a or pass b flagged; reference = 16 agreed + fuxa, froxlor validateIpAddresses, unleash found in one pass"
|
|
},
|
|
"declared_sites_covered_by_candidates": {
|
|
"plane": 19,
|
|
"of": 20,
|
|
"definition": "agree's metric declared_sites_matched: candidates whose site falls on a declared site, flagged or not (winml has no candidate)"
|
|
}
|
|
},
|
|
"reading": "Detection is lower, not only agreement: plane passes flagged 17 of 20 declared sites in at least one pass against the reference's 19 (budibase read and fuxa getDaqValue were flagged by neither plane pass), and two-pass agreement moved the three froxlor sites (pass b only) into 'disputed'; unleash moved the other way. Cost fell 17% (62% of prompt tokens were cache hits). By the pre-registered rule the increment is NOT MET and the design is revisited before task 9.",
|
|
"dev_token_report": {
|
|
"transcript": "the whole development session (c09e30b2), not the increment alone",
|
|
"unique_chars": "7.3M (~1.81M tokens)",
|
|
"tool_use_input_pct": 45.4,
|
|
"tool_result_pct": 40.1,
|
|
"tool_io_pct": 85.5,
|
|
"previous_tool_io_pct": 90,
|
|
"tool_calls": 2701
|
|
},
|
|
"live_defects_found_by_running_on_ax": [
|
|
"no Substrate WorkerPool",
|
|
"tag-pinned images rejected",
|
|
"ax CLI/server skew",
|
|
"unknown-field rejection (atespace, commit)",
|
|
"server-set fields in AX_TASK_YAML",
|
|
"tasks created Suspended; no Completed phase",
|
|
"runner exited after the command",
|
|
"golden-snapshot boot ran the command",
|
|
"egress gateway not deployed; deny-by-default per-actor policy",
|
|
"gateway dials the address the actor dialled",
|
|
"workspace clone before the actor was running",
|
|
"32 KB env limit on inlined inputs",
|
|
"63-byte template name limit"
|
|
],
|
|
"correction": "2026-09-30: the first version of this record read agree's declared_sites_matched (coverage by candidates, 19) as 'found in either pass' and concluded detection was unchanged; recounted from agreed.json rows it is 17 vs the reference's 19."
|
|
}
|