marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert: JavaScript one-start build did not deliver in the push path (M2 11.6)

Browse Source 
Reverts f04a839 on its own measurement. The seven-class profile rerun recorded
joern_parse_census_incomplete on 12 transactions: the faster build was rejected by
its own census gate and the frontend plus overlay pass ran as well, so those
transactions paid two engine starts instead of one. Every workload kept the same
p50 and the verdict stayed NO-GO.

The cause is an asymmetric gate. The frontend path is not census-gated at build
time, so whatever it produces is accepted, while the joern-parse path was gated and
the same shortfall rejected it. Holding the new path to a standard the old one
never has to meet guarantees the fallback fires. The probe tree and the
materialized push snapshot also differ, so equivalence on the checkout did not
transfer to the real path.

What still stands is the underlying timing fact, measured under the production
retention environment on identical graphs, same 44 files, 579 methods, file names,
overlays, Gruntfile and eighteen test files: 11.0 s against 21.0 s. JavaScript does
pay two engine starts where every other language pays one. A future attempt needs a
gate both paths are held to, or a direct comparison against what the frontend would
produce, and must be measured in the push path rather than on a checkout.

The profile also names the remaining cost plainly: a changed-code transaction
spends its budget before any query runs, head build 25.2 s of a 30 s deadline with
query time 0.0 s. Building one new graph per push is the cost, not querying it.

1,401 tests passed, nine skipped. Rollout stays NO-GO.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-17 09:04:44 +02:00
co-authored by Claude Opus 5
parent f04a839181
commit ed0f12b593
5 changed files with 155 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/measurements/2026-09-17-javascript-build-path.json
+5 -1
View File
@@ -146,5 +146,9 @@
"limits": [
"One pinned JavaScript tree. PHP keeps the frontend path, which it needs for exclusions and retries.",
"A build saving is not the latency gate: the profile rerun decides that."
]
],
"outcome": "REVERTED",
"why_reverted": "Measured in the real push path the change did not deliver and cost an extra engine start. The seven-class profile rerun recorded joern_parse_census_incomplete on 12 transactions, so the faster path was rejected by its own census gate and the frontend plus overlay pass ran as well. Every workload stayed at the same p50 and the verdict stayed NO-GO.",
"root_cause_of_the_failed_gate": "The gate is asymmetric. The frontend path is not census-gated at build time, so whatever it produces is accepted; the joern-parse path was gated, so the same shortfall rejected it. Holding the new path to a standard the old path never has to meet guarantees the fallback fires. The probe tree and the materialized push snapshot also differ, so equivalence on the checkout did not transfer.",
"what_still_stands": "The timing difference is real and unexplained by anything else: 11.0 s against 21.0 s for identical graphs under the production retention environment, same 44 files, 579 methods, file names, overlays, Gruntfile and eighteen test files. JavaScript genuinely pays two engine starts where other languages pay one. A future attempt needs a gate both paths are held to, or a comparison against what the frontend would produce."
}
benchmarks/measurements/2026-09-17-session-runtime-profile-after-build-change.json
+136
View File
@@ -0,0 +1,136 @@
{
"workloads": {
"identical-tip": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 23.848443167982623,
"p95_seconds": 23.990965132950805,
"timeouts": 0,
"completed_targets": 0,
"expected_targets": 3,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0,
"pass": false
},
"function-edit": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 30.669465193990618,
"p95_seconds": 30.692176087992266,
"timeouts": 3,
"completed_targets": 0,
"expected_targets": 3,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0.6921760879922658,
"pass": false
},
"dependency-edit": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 30.696090501034632,
"p95_seconds": 30.741624664980918,
"timeouts": 3,
"completed_targets": 0,
"expected_targets": 3,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0.7416246649809182,
"pass": false
},
"configuration-edit": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 30.67598664702382,
"p95_seconds": 30.68519904196728,
"timeouts": 3,
"completed_targets": 0,
"expected_targets": 3,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0.6851990419672802,
"pass": false
},
"cold": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 30.65046051493846,
"p95_seconds": 30.686437385971658,
"timeouts": 3,
"completed_targets": 0,
"expected_targets": 3,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0.6864373859716579,
"pass": false
},
"growth": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 30.951075186021626,
"p95_seconds": 30.9925197820412,
"timeouts": 3,
"completed_targets": 0,
"expected_targets": 3,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0.9925197820411995,
"pass": false
},
"multi-ref": {
"population": 3,
"observed": 3,
"missing": 0,
"latency_samples": 3,
"p50_seconds": 30.678902502986602,
"p95_seconds": 30.69903396104928,
"timeouts": 3,
"completed_targets": 0,
"expected_targets": 6,
"complete_target_cases": 0,
"completion": 0.0,
"complete_coverage_cases": 0,
"cancellation_overrun_seconds": 0.6990339610492811,
"pass": false
}
},
"runtime_verdict": "NO-GO",
"rollout": "NO-GO",
"qualification_limit": "Development NodeGoat, n=3 per workload; independent quality and multi-project performance not established.",
"missing_workloads": [],
"profile_sha256": "26dee90b715f6944651b5022bda573e1bdbeb6a9ea1249c0191c44506fb84d3b",
"priming_completed": false,
"experiment": "session-runtime-profile-v2",
"milestone": "M2 task 11.6",
"bundle": {
"path": "2026-09-17-session-runtime-profile-after-build-change.json.gz",
"sha256": "c83474fbf051156ae1d1f6565001e25d724f2588146dd5114cb00da686e3c63c",
"files": 108
},
"change_under_test": "JavaScript built by joern-parse in one engine start, gated on the census; since reverted.",
"diagnostics": {
"joern_parse_census_incomplete": 12
},
"finding": "The change did not deliver in the real push path. Its own census gate rejected the faster build on 12 transactions, so the frontend and overlay pass ran as well and the transaction paid two engine starts instead of one. Every workload kept the same p50 and the verdict stayed NO-GO, so it was reverted.",
"cause": "An asymmetric gate. The frontend path is not census-gated at build time, so whatever it produces is accepted, while the joern-parse path was gated and the same shortfall rejected it.",
"remaining_cost": "A changed-code transaction spends its whole budget before any query runs: head build 25.2 s of a 30 s deadline with query time 0.0 s. Building one new graph per push is the cost, not querying it."
}
benchmarks/measurements/2026-09-17-session-runtime-profile-after-build-change.json.gz
BIN
View File
Binary file not shown.