mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
ed0f12b593
Reverts f04a839 on its own measurement. The seven-class profile rerun recorded
joern_parse_census_incomplete on 12 transactions: the faster build was rejected by
its own census gate and the frontend plus overlay pass ran as well, so those
transactions paid two engine starts instead of one. Every workload kept the same
p50 and the verdict stayed NO-GO.
The cause is an asymmetric gate. The frontend path is not census-gated at build
time, so whatever it produces is accepted, while the joern-parse path was gated and
the same shortfall rejected it. Holding the new path to a standard the old one
never has to meet guarantees the fallback fires. The probe tree and the
materialized push snapshot also differ, so equivalence on the checkout did not
transfer to the real path.
What still stands is the underlying timing fact, measured under the production
retention environment on identical graphs, same 44 files, 579 methods, file names,
overlays, Gruntfile and eighteen test files: 11.0 s against 21.0 s. JavaScript does
pay two engine starts where every other language pays one. A future attempt needs a
gate both paths are held to, or a direct comparison against what the frontend would
produce, and must be measured in the push path rather than on a checkout.
The profile also names the remaining cost plainly: a changed-code transaction
spends its budget before any query runs, head build 25.2 s of a 30 s deadline with
query time 0.0 s. Building one new graph per push is the cost, not querying it.
1,401 tests passed, nine skipped. Rollout stays NO-GO.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
155 lines
7.2 KiB
JSON
155 lines
7.2 KiB
JSON
{
|
|
"experiment": "javascript-build-path-v1",
|
|
"root": "/cache/nodegoat/c5cb68a7084e",
|
|
"paths": [
|
|
{
|
|
"path": "frontend_then_overlay",
|
|
"frontend_seconds": 6.20875704404898,
|
|
"overlay_seconds": 14.808171501965262,
|
|
"total_seconds": 21.016928546014242,
|
|
"frontend_exit": 0,
|
|
"overlay_exit": 0,
|
|
"census": {
|
|
"files": "44",
|
|
"methods": "579",
|
|
"calls": "4134",
|
|
"file_names": [
|
|
"Gruntfile.js",
|
|
"app/assets/js/chart/chart-data-morris.js",
|
|
"app/assets/js/tour/redirects-steps.js",
|
|
"app/data/allocations-dao.js",
|
|
"app/data/benefits-dao.js",
|
|
"app/data/contributions-dao.js",
|
|
"app/data/memos-dao.js",
|
|
"app/data/profile-dao.js",
|
|
"app/data/research-dao.js",
|
|
"app/data/user-dao.js",
|
|
"app/routes/allocations.js",
|
|
"app/routes/benefits.js",
|
|
"app/routes/contributions.js",
|
|
"app/routes/error.js",
|
|
"app/routes/index.js",
|
|
"app/routes/memos.js",
|
|
"app/routes/profile.js",
|
|
"app/routes/research.js",
|
|
"app/routes/session.js",
|
|
"app/routes/tutorial.js",
|
|
"artifacts/db-reset.js",
|
|
"config/config.js",
|
|
"config/env/all.js",
|
|
"config/env/development.js",
|
|
"config/env/production.js",
|
|
"config/env/test.js",
|
|
"server.js",
|
|
"test/e2e/integration/allocations_spec.js",
|
|
"test/e2e/integration/benefits_spec.js",
|
|
"test/e2e/integration/contributions_spec.js",
|
|
"test/e2e/integration/dashboard_spec.js",
|
|
"test/e2e/integration/general_spec.js",
|
|
"test/e2e/integration/learn_spec.js",
|
|
"test/e2e/integration/login_spec.js",
|
|
"test/e2e/integration/logout_spec.js",
|
|
"test/e2e/integration/memos_spec.js",
|
|
"test/e2e/integration/profile_spec.js",
|
|
"test/e2e/integration/research_spec.js",
|
|
"test/e2e/integration/signup_spec.js",
|
|
"test/e2e/integration/tutorial_spec.js",
|
|
"test/e2e/plugins/index.js",
|
|
"test/e2e/support/commands.js",
|
|
"test/e2e/support/index.js",
|
|
"test/security/profile-test.js"
|
|
],
|
|
"overlays": "base,controlflow,typerel,callgraph,dataflowOss",
|
|
"maxHeapMB": "2048"
|
|
}
|
|
},
|
|
{
|
|
"path": "joern_parse_only",
|
|
"total_seconds": 10.995579135953449,
|
|
"exit": 0,
|
|
"stderr_tail": "Picked up JAVA_TOOL_OPTIONS: -Xmx2048m\n2026-09-17 06:41:16.285 WARN CfgCreator$ Using order fallback for try statement finally body\n",
|
|
"census": {
|
|
"files": "44",
|
|
"methods": "579",
|
|
"calls": "4134",
|
|
"file_names": [
|
|
"Gruntfile.js",
|
|
"app/assets/js/chart/chart-data-morris.js",
|
|
"app/assets/js/tour/redirects-steps.js",
|
|
"app/data/allocations-dao.js",
|
|
"app/data/benefits-dao.js",
|
|
"app/data/contributions-dao.js",
|
|
"app/data/memos-dao.js",
|
|
"app/data/profile-dao.js",
|
|
"app/data/research-dao.js",
|
|
"app/data/user-dao.js",
|
|
"app/routes/allocations.js",
|
|
"app/routes/benefits.js",
|
|
"app/routes/contributions.js",
|
|
"app/routes/error.js",
|
|
"app/routes/index.js",
|
|
"app/routes/memos.js",
|
|
"app/routes/profile.js",
|
|
"app/routes/research.js",
|
|
"app/routes/session.js",
|
|
"app/routes/tutorial.js",
|
|
"artifacts/db-reset.js",
|
|
"config/config.js",
|
|
"config/env/all.js",
|
|
"config/env/development.js",
|
|
"config/env/production.js",
|
|
"config/env/test.js",
|
|
"server.js",
|
|
"test/e2e/integration/allocations_spec.js",
|
|
"test/e2e/integration/benefits_spec.js",
|
|
"test/e2e/integration/contributions_spec.js",
|
|
"test/e2e/integration/dashboard_spec.js",
|
|
"test/e2e/integration/general_spec.js",
|
|
"test/e2e/integration/learn_spec.js",
|
|
"test/e2e/integration/login_spec.js",
|
|
"test/e2e/integration/logout_spec.js",
|
|
"test/e2e/integration/memos_spec.js",
|
|
"test/e2e/integration/profile_spec.js",
|
|
"test/e2e/integration/research_spec.js",
|
|
"test/e2e/integration/signup_spec.js",
|
|
"test/e2e/integration/tutorial_spec.js",
|
|
"test/e2e/plugins/index.js",
|
|
"test/e2e/support/commands.js",
|
|
"test/e2e/support/index.js",
|
|
"test/security/profile-test.js"
|
|
],
|
|
"overlays": "base,controlflow,typerel,callgraph,dataflowOss",
|
|
"maxHeapMB": "2048"
|
|
}
|
|
}
|
|
],
|
|
"source_files": 50,
|
|
"source_bytes": 367111,
|
|
"retention": {
|
|
"frontend_has_gruntfile": true,
|
|
"joern_parse_has_gruntfile": true,
|
|
"frontend_test_files": 18,
|
|
"joern_parse_test_files": 18,
|
|
"only_in_frontend": [],
|
|
"only_in_joern_parse": []
|
|
},
|
|
"comparison": {
|
|
"same_files": true,
|
|
"same_methods": true,
|
|
"same_overlays": true,
|
|
"same_file_names": true,
|
|
"seconds_saved": 10.02
|
|
},
|
|
"milestone": "M2 task 11.6",
|
|
"instrument_note": "A first run without OUSAST_INCLUDE_TESTS and OUSAST_INCLUDE_BUILD_CONFIGS reported 25 files from both paths and appeared to show equivalence. It did not: both runs dropped the retained files, so the comparison said nothing about the retention adaptation the frontend path exists for. An existing control caught the change built on it. This run sets the production environment.",
|
|
"decision": "JavaScript now builds through joern-parse, one engine start including overlays, and falls back to the declared frontend plus overlay pass whenever the census is short. Both paths hold the same 44 files, 579 methods, file names, overlays, the Gruntfile and all eighteen first-party test files.",
|
|
"limits": [
|
|
"One pinned JavaScript tree. PHP keeps the frontend path, which it needs for exclusions and retries.",
|
|
"A build saving is not the latency gate: the profile rerun decides that."
|
|
],
|
|
"outcome": "REVERTED",
|
|
"why_reverted": "Measured in the real push path the change did not deliver and cost an extra engine start. The seven-class profile rerun recorded joern_parse_census_incomplete on 12 transactions, so the faster path was rejected by its own census gate and the frontend plus overlay pass ran as well. Every workload stayed at the same p50 and the verdict stayed NO-GO.",
|
|
"root_cause_of_the_failed_gate": "The gate is asymmetric. The frontend path is not census-gated at build time, so whatever it produces is accepted; the joern-parse path was gated, so the same shortfall rejected it. Holding the new path to a standard the old path never has to meet guarantees the fallback fires. The probe tree and the materialized push snapshot also differ, so equivalence on the checkout did not transfer.",
|
|
"what_still_stands": "The timing difference is real and unexplained by anything else: 11.0 s against 21.0 s for identical graphs under the production retention environment, same 44 files, 579 methods, file names, overlays, Gruntfile and eighteen test files. JavaScript genuinely pays two engine starts where other languages pay one. A future attempt needs a gate both paths are held to, or a comparison against what the frontend would produce."
|
|
}
|