mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5
ffbd864195
The unchanged seven-class profile, three samples each, with the engine session enabled. Every changed-code workload still times out at the 30-second deadline, and identical-tip moved the wrong way: p50 rose from 18.5 s to 24.2 s. The cause is in the artifacts rather than inferred. Thirteen transactions report session_unavailable:session_startup_timeout, so under the real deadline the session usually never starts and everything falls back to the disposable path. Startup is 9 to 13 seconds, about a third of the whole budget, and it is spent before any analysis happens. Where a session did start, on cached-graph transactions whose remaining work is small, it cost more than it saved. That is the honest shape of the result: a transaction-owned session amortizes engine startup across the invocations inside one transaction, and a warm changed-code push does not have enough of them. It helps a long development-budget transaction, 326 s to 243 s measured earlier, and cannot help a 30-second one. Correctness is unchanged throughout: the same three pinned revisions through the production rules with the session enabled reproduce the M1 verdict exactly. Group 11 as designed cannot reach the gate, so the dominant remaining cost is named and the decision is raised rather than worked around. Task 11.6 records the candidates, since two of them change the product's shape: a session shared across transactions is a resident local service, which the accepted milestone spec does not imply, and analysing less is a scope change rather than a speed change. The 30-second and 2-second gates stay unchanged. Rollout stays NO-GO. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
148 lines
5.2 KiB
JSON
148 lines
5.2 KiB
JSON
{
|
|
"workloads": {
|
|
"identical-tip": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 24.163701707031578,
|
|
"p95_seconds": 24.394183435942978,
|
|
"timeouts": 0,
|
|
"completed_targets": 0,
|
|
"expected_targets": 3,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 0,
|
|
"pass": false
|
|
},
|
|
"function-edit": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 30.771415333962068,
|
|
"p95_seconds": 30.935717942891642,
|
|
"timeouts": 3,
|
|
"completed_targets": 0,
|
|
"expected_targets": 3,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 0.9357179428916425,
|
|
"pass": false
|
|
},
|
|
"dependency-edit": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 30.8792759970529,
|
|
"p95_seconds": 30.88528792699799,
|
|
"timeouts": 3,
|
|
"completed_targets": 0,
|
|
"expected_targets": 3,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 0.8852879269979894,
|
|
"pass": false
|
|
},
|
|
"configuration-edit": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 30.830487370025367,
|
|
"p95_seconds": 30.94464882405009,
|
|
"timeouts": 3,
|
|
"completed_targets": 0,
|
|
"expected_targets": 3,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 0.9446488240500912,
|
|
"pass": false
|
|
},
|
|
"cold": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 30.70144796196837,
|
|
"p95_seconds": 30.723933225031942,
|
|
"timeouts": 3,
|
|
"completed_targets": 0,
|
|
"expected_targets": 3,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 0.7239332250319421,
|
|
"pass": false
|
|
},
|
|
"growth": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 31.13915658602491,
|
|
"p95_seconds": 31.186890079989098,
|
|
"timeouts": 3,
|
|
"completed_targets": 0,
|
|
"expected_targets": 3,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 1.186890079989098,
|
|
"pass": false
|
|
},
|
|
"multi-ref": {
|
|
"population": 3,
|
|
"observed": 3,
|
|
"missing": 0,
|
|
"latency_samples": 3,
|
|
"p50_seconds": 30.744394473033026,
|
|
"p95_seconds": 30.833070866996422,
|
|
"timeouts": 3,
|
|
"completed_targets": 0,
|
|
"expected_targets": 6,
|
|
"complete_target_cases": 0,
|
|
"completion": 0.0,
|
|
"complete_coverage_cases": 0,
|
|
"cancellation_overrun_seconds": 0.8330708669964224,
|
|
"pass": false
|
|
}
|
|
},
|
|
"runtime_verdict": "NO-GO",
|
|
"rollout": "NO-GO",
|
|
"qualification_limit": "Development NodeGoat, n=3 per workload; independent quality and multi-project performance not established.",
|
|
"missing_workloads": [],
|
|
"profile_sha256": "077254b581db671f6732f72bf15efc7f12d3b3c72d9de16be3033624860a2989",
|
|
"priming_completed": false,
|
|
"bundle": {
|
|
"path": "2026-09-17-session-runtime-profile.json.gz",
|
|
"sha256": "34d0b23e5abd052bdd060a2db922f8ad2eb21d0229f81368c8205842e0934ced",
|
|
"files": 108
|
|
},
|
|
"experiment": "session-runtime-profile-v1",
|
|
"milestone": "M2 task 11.5",
|
|
"transport": "engine session enabled via OUSAST_ENGINE_SESSION=1",
|
|
"session_outcomes": {
|
|
"session_unavailable:session_startup_timeout": 13,
|
|
"session_define_failed:census": 1
|
|
},
|
|
"finding": "Under the real 30-second deadline the session almost never starts: 13 transactions recorded session_startup_timeout and fell back to the disposable path. Session startup measured 9 to 13 seconds, roughly a third of the whole budget, and it is spent before any analysis. Where a session did start, on identical-tip transactions whose graphs are cached and whose remaining work is small, it made things worse: p50 rose from 18.5 s in the pre-session profile to 24.2 s. A transaction-owned session relocates engine startup rather than removing it, and it can only amortize that cost across many engine invocations, which a warm changed-code transaction does not have.",
|
|
"comparison_with_pre_session_profile": {
|
|
"identical-tip p50": {
|
|
"before": 18.5,
|
|
"after": 24.2
|
|
},
|
|
"changed-code workloads": "all still time out at the 30-second deadline, before and after"
|
|
},
|
|
"correctness": "Unchanged. The same three pinned revisions through the production rules with the session enabled reproduce the M1 verdict exactly: benchmarks/measurements/2026-09-17-nodegoat-session-transport.json.",
|
|
"limits": [
|
|
"Development NodeGoat, three samples per workload; not independent qualification.",
|
|
"Priming completed in 236 s without completing its target, so warm graph reuse is partial."
|
|
]
|
|
}
|