diff --git a/.kiro/specs/contributor-scan/pre-push-runtime-followup.md b/.kiro/specs/contributor-scan/pre-push-runtime-followup.md new file mode 100644 index 0000000..0fe74f9 --- /dev/null +++ b/.kiro/specs/contributor-scan/pre-push-runtime-followup.md @@ -0,0 +1,43 @@ +# Runtime follow-up from pre-push-safety-net task 8.2 + +Status: frozen measurement complete, runtime NO-GO; investigation routed to the existing engine/runtime +owner. This note does not change contributor-scan task approvals or detector semantics. +Final scorecard and exact instrument identities are linked below. + +The NodeGoat production transaction uses the existing evidence ranker and full tracked +first-party graph. Initial packaged process receipts show approximately 5 seconds for +the JavaScript frontend, 14 seconds for overlays and 12 seconds for census before the +first query. Each query invokes another JVM. Both comparison sides retain independent +graph/answer identities. These are observed single-stage costs, not p95 estimates. + +The existing engine boundary owns frontend execution, overlays, graph census, queries, +process cancellation and validated graph leases. Investigate batching census and query +work into an engine session, amortizing JVM startup, and reusing complete immutable +base-side artifacts. Measure the additional costs of graph parsing, overlay work and +loading separately. A long-lived local engine would need request deadlines, isolated +leases, reliable cancellation/recovery and resource limits before it can replace the +current disposable subprocess lifecycle. + +Any proposed change must preserve named source/census receipts, actual selected/deferred +question identities, first-party-only graph contents, exact base/head comparisons and +fresh-answer census validation. Compare evidence against cold execution. Keep the +30-second warm p95 and 2-second cancellation/report gates unchanged. Differential graph +mutation remains a separate experiment requiring cold-equivalence evidence. Ranker +retuning or shrinking the evaluation population cannot repair fixed engine startup cost. + +The initial long-budget NodeGoat priming transaction finished in 224.47 seconds with +incomplete coverage. Its head partition records 44 input paths and 95,411 source bytes, +but the scan reports `cpg_empty` and `partition_file_census_incomplete`; the declared +contributions handler remains an unresolved file-level question. No qualified graph +or answer hit is established for the following identical comparison. This is a named +instrument/coverage gap, not evidence that the target is safe or that the ranker missed +a proven candidate. Reconcile raw frontend/census filenames with the 44 input paths, +including the explicit test-retention policy, before attributing this to cache identity +or detector semantics. Keep this investigation separate from JVM startup optimization. + +Final task 8.2 receipts: `benchmarks/measurements/2026-09-13-nodegoat-feasibility.json` +and its compressed full evidence bundle. All 21 deadline-bound transactions timed out, +0/24 declared target checks completed, observed workload p95 30.631–30.912 seconds. +Every measured artifact matches the frozen installed semantics. No source, engine, +query, fact or ranking change was made during this run. The follow-up remains necessary +before rollout; the evaluation task does not authorize weakening its gates. diff --git a/.kiro/specs/pre-push-safety-net/implementation-group-8.md b/.kiro/specs/pre-push-safety-net/implementation-group-8.md index 8fe43bf..c3da06b 100644 --- a/.kiro/specs/pre-push-safety-net/implementation-group-8.md +++ b/.kiro/specs/pre-push-safety-net/implementation-group-8.md @@ -35,3 +35,74 @@ missing-base, hook-exit and ordinary-scan regressions. - CLAIM: Task 8.1 integration regression coverage delivered. - EVIDENCE: `/tmp/group8-1-focused.log`: 3 passed; `/tmp/group8-1-full.log`: 1299 passed, 9 skipped; static checks exit 0. - GAPS: These controlled failures do not qualify real detector capabilities or representative latency. + +## Task 8.2 brief + +Requirements 4.1, 4.2, 6.2, 6.3, 6.5 and 7.3; evaluation/runtime integration boundary. +Freeze full pinned NodeGoat histories before measuring: three distinct function edits, +dependency edits, configuration edits, growth histories and multi-ref transactions, +plus three identical-comparison and three uncached controls. Each edit repetition has +a distinct tree, avoiding accidental identical-tip reuse. Retain all 21 transactions, +exact source byte receipt, installed semantics, hardware, selected/deferred counts, +expected target checks, stage process receipts and whole CLI elapsed time. A separate +600-second priming run is diagnostic and cannot replace a 30-second gate sample. + +Nearest-rank p50/p95 include timeout durations and report small sample counts explicitly. +A missing artifact or abnormal CLI exit is an instrument error, never a clean result. +Source/vendor/context/coverage gaps remain gaps. NodeGoat remains development data; +this runtime experiment cannot qualify independent alert precision or broad production +performance. Engine lifecycle costs route to contributor-scan's runtime follow-up. + +### Task 8.2 measured outcome + +Packaged image `af5b896b406fe98acb3d00bb2180a7549febd6dd285f0b63087538118e52e1f4`, +network disabled, UID 1000, 3 GiB memory limit; CPU/affinity and installed identities +are in `benchmarks/measurements/2026-09-13-nodegoat-feasibility.json`. All 21 artifacts +match the frozen core/facts/queries/policy/runtime/semantics. All 24 declared target +checks remain incomplete. Priming took 224.47 seconds and did not qualify graph/answer +reuse; subsequent cache attempts establish discovery reuse only. + +| Workload | n | p50 seconds | p95 seconds | Timeouts | Completed target checks | +|---|---:|---:|---:|---:|---:| +| Identical comparison | 3 | 30.635 | 30.644 | 3 | 0/3 | +| Distinct function edits | 3 | 30.611 | 30.663 | 3 | 0/3 | +| Dependency edits | 3 | 30.627 | 30.712 | 3 | 0/3 | +| Configuration edits | 3 | 30.606 | 30.728 | 3 | 0/3 | +| Cold | 3 | 30.609 | 30.631 | 3 | 0/3 | +| Growth | 3 | 30.894 | 30.912 | 3 | 0/3 | +| Multi-ref | 3 | 30.652 | 30.692 | 3 | 0/6 | + +These are observed timings of the declared attempts, not proof of a qualified warm +graph path. All timeouts remain in the population. Cancellation/reporting meets the +2-second allowance in these samples; latency and completion do not meet the joint gate. +Runtime and rollout verdict: **NO-GO**. The n=3 quantiles do not establish multi-project +production performance. The full 116-file receipt bundle is 1,254,241 bytes compressed, +SHA-256 `3174bfc0bd39e10b685d06d8bc28ab39450368f1a441e64bbb5a27a256bf13c6`. +Decompression was compared to every original file. Both measured instrument bytes and +the original frozen scorecard are retained; later scorer hardening preserves the +measurements and adds missing-workload and multi-ref-denominator checks. + +Owner routing: `../contributor-scan/pre-push-runtime-followup.md` separates raw named +census reconciliation from frontend/overlay/JVM/query lifecycle optimization. Do not +retune the ranker or weaken the population/gates to compensate. + +## Status Report +- STATUS: READY_FOR_REVIEW +- TASK: 8.2 +- RED_PHASE_OUTPUT: `/tmp/group8-2-red.log` (missing scorer) and `/tmp/group8-2-flag-off.log` (disabled scorer). ON passed; switch removed, seven final scorer tests passed. +- TESTS_RUN: full suite 1306 passed, 9 skipped in 117.63s; ruff, formatting, mypy and diff checks pass. Frozen packaged measurement completed with exit 1 and full NO-GO scorecard, no missing artifacts or abnormal CLI exits. + +## Review Verdict +- VERDICT: APPROVED +- TASK: 8.2 +- MECHANICAL_RESULTS: actual harness, fixtures, scorecard and receipt bundle reviewed. Input bytes/pins, distinct edit histories, all 21 runs, stage exits, fixed target denominators and source/runtime identity checks retained. Invalid numbers, duplicate/unplanned rows, missing workload classes and missing cases cannot qualify runtime. No production/ranker/fact changes. +- FINDINGS: Initial warm-graph precondition failed; the report explicitly limits reuse claims. The unchanged runtime gates fail. Engine lifecycle and named census follow-up routed to contributor-scan. +- RED phase: VERIFIED. +- SUMMARY: Reproducible runtime evaluation is complete with a measured NO-GO result. + +## Verification Result +- STATUS: VERIFIED +- CLAIM_TYPE: TASK +- CLAIM: Task 8.2 completed its prescribed measurement and owner routing; rollout remains NO-GO. +- EVIDENCE: Versioned scorecard and 116-file verified bundle; seven scorer controls and full regression suite pass. +- GAPS: No qualified warm graph path or independent capability eligibility established. These failed gates cannot authorize rollout. diff --git a/.kiro/specs/pre-push-safety-net/spec.json b/.kiro/specs/pre-push-safety-net/spec.json index c9a53f3..0586e7a 100644 --- a/.kiro/specs/pre-push-safety-net/spec.json +++ b/.kiro/specs/pre-push-safety-net/spec.json @@ -116,9 +116,10 @@ "7.1", "7.2", "7.3", - "8.1" + "8.1", + "8.2" ], - "remaining_tasks": 3, + "remaining_tasks": 2, "verification_record": "implementation-group-8.md", "verification_records": [ "implementation-group-1.md", @@ -132,6 +133,6 @@ ], "next_group": 8, "blocked_tasks": [], - "feasibility": "NO-GO for rollout: compatible identical-revision reuse verified; representative changed-code p95/completion and independent alert eligibility remain unmet." + "feasibility": "NO-GO: all 21 frozen NodeGoat transactions timed out; 0/24 declared target checks completed. Cancellation/reporting remained below 2 seconds. Priming did not qualify graph reuse; named census and engine lifecycle work routed to contributor-scan." } } diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index fb2e2d5..d14a02f 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -218,7 +218,7 @@ independent capability qualification. Groups 7–8 remain pending; rollout remai - Done when the integration suite covers the critical flows through the real CLI/backend seams and preserves ordinary scan behavior. - _Boundary: Push runner integration tests_ - _Requirements: 1.1, 1.2, 1.3, 1.4, 2.2, 2.3, 4.2, 4.4, 5.2, 5.5, 7.1, 7.4, 8.2_ -- [ ] 8.2 Measure realistic cache and deadline feasibility +- [x] 8.2 Measure realistic cache and deadline feasibility - Measure identical-tip reuse, a warm one-function edit, dependency/configuration edits, cold starts, repository growth and multi-ref workloads on declared hardware. - If reuse correctness passes but changed-code checks time out, identify preparation, frontend/overlay/census and query costs separately and route the remaining optimization to its owning boundary before rollout GO. Preserve the declared gates and do not substitute identical-tip latency. - Publish p50/p95, timeouts, cancellation allowance, completed/deferred population and stage costs; the denominator is fixed before ranker selection. diff --git a/benchmarks/measurements/2026-09-13-nodegoat-feasibility.json b/benchmarks/measurements/2026-09-13-nodegoat-feasibility.json new file mode 100644 index 0000000..06c0b26 --- /dev/null +++ b/benchmarks/measurements/2026-09-13-nodegoat-feasibility.json @@ -0,0 +1,395 @@ +{ + "workloads": { + "identical-tip": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.634708732017316, + "p95_seconds": 30.644362346967682, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 3, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.6443623469676822, + "pass": false + }, + "function-edit": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.610851514036767, + "p95_seconds": 30.663211637060158, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 3, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.663211637060158, + "pass": false + }, + "dependency-edit": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.6267360819038, + "p95_seconds": 30.712326238048263, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 3, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.7123262380482629, + "pass": false + }, + "configuration-edit": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.606312285992317, + "p95_seconds": 30.72775209392421, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 3, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.7277520939242095, + "pass": false + }, + "cold": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.608712024986744, + "p95_seconds": 30.63113492797129, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 3, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.6311349279712886, + "pass": false + }, + "growth": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.893849348067306, + "p95_seconds": 30.912386535899714, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 3, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.9123865358997136, + "pass": false + }, + "multi-ref": { + "population": 3, + "observed": 3, + "missing": 0, + "latency_samples": 3, + "p50_seconds": 30.65246668900363, + "p95_seconds": 30.691843352047727, + "timeouts": 3, + "completed_targets": 0, + "expected_targets": 6, + "complete_target_cases": 0, + "completion": 0.0, + "complete_coverage_cases": 0, + "cancellation_overrun_seconds": 0.6918433520477265, + "pass": false + } + }, + "runtime_verdict": "NO-GO", + "rollout": "NO-GO", + "qualification_limit": "Development NodeGoat, n=3 per workload; independent quality and multi-project performance not established.", + "missing_workloads": [], + "schema_version": 1, + "experiment": "group8-nodegoat-runtime-v1", + "image": "sha256:af5b896b406fe98acb3d00bb2180a7549febd6dd285f0b63087538118e52e1f4", + "profile_sha256": "488232afd5e67e92ab9407980a0f905cbcc0c6e28182e947956b6b6cf0666be5", + "provenance": { + "repository": "/results/nodegoat-v1/objects.git", + "engine_runtime": "2b415139532d8d3a0c6e46ecdac26cd3d792efc2bae441f0b7e39bb28ec5d16e", + "engine": "joern-4.0.625", + "facts": "32edb6e1a78f1afb08ac4248cd436be16663e0d90dc1b3aaf948f2ce49a5a8f0", + "graph_layout": "060428173f47cf33bdf5e532d4069feb598ba0e8c7ea30a60888c8e4cfb37a42", + "queries": "7bf090e053331f03d04c9290c8a593d00efd4d390f4b77b8382c2befc47996cc", + "policy": "5ea302fdc605e23676e09765890c7b491c89a69c2797feb0a5aa3346ddce81ef", + "core": "36ac9a045b8b8ad943eea1f511924caae54750a9d5d45da776ebefc460f08eab", + "semantics": "f652da2ca5891b6e86a242318357a36f4e3b95529fea596f7d064ef35b2d6cd2", + "ranking_mode": "evidence", + "model": "disabled", + "capabilities": "empty_registry_pending_task_8.3", + "cache": "cold_no_reuse" + }, + "hardware": { + "platform": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39", + "cpu_affinity": [ + 0, + 1, + 2, + 3 + ], + "cpuinfo": "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 158\nmodel name\t: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\nstepping\t: 10\nmicrocode\t: 0xfa\ncpu MHz\t\t: 3192.000\ncache size\t: 16384 KB\nphysical id\t: 0\nsiblings\t: 1\ncore id\t\t: 0\ncpu cores\t: 1\napicid\t\t: 0\ninitial apicid\t: 0\nfpu\t\t: yes\nfpu_exception\t: yes\ncpuid level\t: 22\nwp\t\t: yes\nflags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault ssbd ibrs ibpb stibp tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat vnmi umip md_clear flush_l1d arch_capabilities\nvmx flags\t: vnmi preemption_timer invvpid ept_x_only ept_ad ept_1gb flexpriority tsc_offset vtpr mtf vapic ept vpid unrestricted_guest shadow_vmcs pml\nbugs\t\t: cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs srbds mmio_stale_data retbleed bhi ibpb_no_ret its\nbogomips\t: 6384.00\nclflush size\t: 64\ncache_alignment\t: 64\naddress sizes\t: 39 bits physical, 48 bits virtual\npower management:\n\nprocessor\t: 1\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 158\nmodel name\t: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\nstepping\t: 10\nmicrocode\t: 0xfa\ncpu MHz\t\t: 3192.000\ncache size\t: 16384 KB\nphysical id\t: 1\nsiblings\t: 1\ncore id\t\t: 0\ncpu cores\t: 1\napicid\t\t: 1\ninitial apicid\t: 1\nfpu\t\t: yes\nfpu_exception\t: yes\ncpuid level\t: 22\nwp\t\t: yes\nflags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault ssbd ibrs ibpb stibp tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat vnmi umip md_clear flush_l1d arch_capabilities\nvmx flags\t: vnmi preemption_timer invvpid ept_x_only ept_ad ept_1gb flexpriority tsc_offset vtpr mtf vapic ept vpid unrestricted_guest shadow_vmcs pml\nbugs\t\t: cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs srbds mmio_stale_data retbleed bhi ibpb_no_ret its\nbogomips\t: 6384.00\nclflush size\t: 64\ncache_alignment\t: 64\naddress sizes\t: 39 bits physical, 48 bits virtual\npower management:\n\nprocessor\t: 2\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 158\nmodel name\t: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\nstepping\t: 10\nmicrocode\t: 0xfa\ncpu MHz\t\t: 3192.000\ncache size\t: 16384 KB\nphysical id\t: 2\nsiblings\t: 1\ncore id\t\t: 0\ncpu cores\t: 1\napicid\t\t: 2\ninitial apicid\t: 2\nfpu\t\t: yes\nfpu_exception\t: yes\ncpuid level\t: 22\nwp\t\t: yes\nflags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault ssbd ibrs ibpb stibp tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat vnmi umip md_clear flush_l1d arch_capabilities\nvmx flags\t: vnmi preemption_timer invvpid ept_x_only ept_ad ept_1gb flexpriority tsc_offset vtpr mtf vapic ept vpid unrestricted_guest shadow_vmcs pml\nbugs\t\t: cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs srbds mmio_stale_data retbleed bhi ibpb_no_ret its\nbogomips\t: 6384.00\nclflush size\t: 64\ncache_alignment\t: 64\naddress sizes\t: 39 bits physical, 48 bits virtual\npower management:\n\nprocessor\t: 3\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 158\nmodel name\t: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\nstepping\t: 10\nmicrocode\t: 0xfa\ncpu MHz\t\t: 3192.000\ncache size\t: 16384 KB\nphysical id\t: 3\nsiblings\t: 1\ncore id\t\t: 0\ncpu cores\t: 1\napicid\t\t: 3\ninitial apicid\t: 3\nfpu\t\t: yes\nfpu_exception\t: yes\ncpuid level\t: 22\nwp\t\t: yes\nflags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault ssbd ibrs ibpb stibp tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat vnmi umip md_clear flush_l1d arch_capabilities\nvmx flags\t: vnmi preemption_timer invvpid ept_x_only ept_ad ept_1gb flexpriority tsc_offset vtpr mtf vapic ept vpid unrestricted_guest shadow_vmcs pml\nbugs\t\t: cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs srbds mmio_stale_data retbleed bhi ibpb_no_ret its\nbogomips\t: 6384.00\nclflush size\t: 64\ncache_alignment\t: 64\naddress sizes\t: 39 bits physical, 48 bits virtual\npower management:\n\n", + "memory_limit": "3221225472" + }, + "input_probe": { + "path": "app/routes/contributions.js", + "bytes": 2485, + "sha256": "196bdeaa22da4cfe8a851b1b932710b2fcedd21f14ed75d70b739ba759a01003" + }, + "sample_count": 21, + "expected_target_checks": 24, + "complete_target_checks": 0, + "priming": { + "id": "lab-prime", + "elapsed_seconds": 224.47236027393956, + "exit_code": 0, + "instrument_error": null, + "timed_out": false, + "target_completed": false, + "completed_targets": 0, + "expected_targets": 1, + "coverage": "incomplete", + "timings": { + "comparison_0_deadline_seconds": 600.0, + "comparison_0_cancellation_allowance_seconds": 2.0, + "comparison_0_resolution_seconds": 0.00692109193187207, + "comparison_0_provenance_seconds": 0.2868837690912187, + "comparison_0_snapshot_seconds": 0.8197841900400817, + "comparison_0_snapshot_reuses": 0, + "comparison_0_discovery_seconds": 6.712452265899628, + "comparison_0_discovery_hits": 0, + "comparison_0_preparation_seconds": 7.551641582977027, + "comparison_0_head_seconds": 102.4524846150307, + "comparison_0_base_seconds": 113.50927831302397, + "comparison_0_admission_seconds": 0.006122270948253572, + "comparison_0_cleanup_seconds": 0.011785765993408859, + "comparison_0_graph_hits": 0, + "comparison_0_query_hits": 0, + "comparison_0_total_seconds": 223.85807027190458, + "total_seconds": 223.87894638301805, + "deadline_seconds": 600.0, + "cancellation_allowance_seconds": 2.0 + }, + "selected": 436, + "deferred": 0, + "stages": [ + { + "stage": "frontend", + "seconds": 4.982722881948575, + "exit_code": 0, + "receipt": true, + "executable": "jssrc2cpg", + "scripts": [] + }, + { + "stage": "overlay", + "seconds": 13.621651508961804, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "overlay.sc" + ] + }, + { + "stage": "census", + "seconds": 12.337514301994815, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "census.sc" + ] + }, + { + "stage": "query", + "seconds": 18.39221175899729, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "taint.sc" + ] + }, + { + "stage": "query", + "seconds": 13.111017563031055, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "dominance.sc" + ] + }, + { + "stage": "query", + "seconds": 12.429657276021317, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "config.sc" + ] + }, + { + "stage": "query", + "seconds": 19.411302927066572, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "taint.sc" + ] + }, + { + "stage": "frontend", + "seconds": 5.707860570051707, + "exit_code": 0, + "receipt": true, + "executable": "jssrc2cpg", + "scripts": [] + }, + { + "stage": "overlay", + "seconds": 15.832738576922566, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "overlay.sc" + ] + }, + { + "stage": "census", + "seconds": 13.896116655902006, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "census.sc" + ] + }, + { + "stage": "query", + "seconds": 18.60998445993755, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "taint.sc" + ] + }, + { + "stage": "query", + "seconds": 14.474358166917227, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "dominance.sc" + ] + }, + { + "stage": "query", + "seconds": 14.400734664988704, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "config.sc" + ] + }, + { + "stage": "query", + "seconds": 22.831895098090172, + "exit_code": 0, + "receipt": true, + "executable": "joern", + "scripts": [ + "taint.sc" + ] + } + ] + }, + "bundle": { + "path": "2026-09-13-nodegoat-feasibility.json.gz", + "sha256": "3174bfc0bd39e10b685d06d8bc28ab39450368f1a441e64bbb5a27a256bf13c6", + "bytes": 1254241, + "files": 116 + }, + "instrument_sha256": "89667b2821bea8d5437a01df1a52e17b596c6b66c132db83e828e444feeabbb8", + "stages_by_workload": { + "identical-tip": { + "frontend": 16.57258843292948, + "overlay": 46.36333934101276, + "census": 21.44899461104069, + "query": 0, + "input_probe": 0 + }, + "function-edit": { + "frontend": 17.311220249161124, + "overlay": 46.13778600492515, + "census": 10.856501498026773, + "query": 0, + "input_probe": 0 + }, + "dependency-edit": { + "frontend": 17.487801696988754, + "overlay": 45.022392168990336, + "census": 11.75914374506101, + "query": 0, + "input_probe": 0 + }, + "configuration-edit": { + "frontend": 17.555444408091716, + "overlay": 46.67416559392586, + "census": 10.313088580965996, + "query": 0, + "input_probe": 0 + }, + "cold": { + "frontend": 16.713468418922275, + "overlay": 45.5736131190788, + "census": 0, + "query": 0, + "input_probe": 0 + }, + "growth": { + "frontend": 19.406804273021407, + "overlay": 42.605494707939215, + "census": 0, + "query": 0, + "input_probe": 0 + }, + "multi-ref": { + "frontend": 18.01197422691621, + "overlay": 46.99900957697537, + "census": 19.20424186089076, + "query": 0, + "input_probe": 0 + } + }, + "limitations": [ + "n=3 per development workload; no independent detector qualification", + "Incomplete priming did not qualify graph/answer reuse; discovery cache is the only established reuse here", + "All timeout samples retained; no warm graph performance established", + "All 21 raw artifact core/fact/query/policy/runtime/semantics identities match the frozen profile", + "Scorer hardened after run to reject invalid samples, missing workload classes and preserve multi-ref target denominators; raw profile, instrument and original scorecard retained unchanged" + ] +} diff --git a/benchmarks/measurements/2026-09-13-nodegoat-feasibility.json.gz b/benchmarks/measurements/2026-09-13-nodegoat-feasibility.json.gz new file mode 100644 index 0000000..87039e2 Binary files /dev/null and b/benchmarks/measurements/2026-09-13-nodegoat-feasibility.json.gz differ diff --git a/benchmarks/push/README.md b/benchmarks/push/README.md index 34e28e9..9d05e05 100644 --- a/benchmarks/push/README.md +++ b/benchmarks/push/README.md @@ -158,3 +158,33 @@ The output directory must be empty and writable by the image's nonroot user. Col 30-second deadline and 2-second cancellation allowance. Default normal capabilities remain empty. A passing reuse control cannot authorize rollout or turn unresolved coverage into clean analysis; groups 7–8 still own the real interface and joint evaluation. + +### Frozen runtime feasibility (task 8.2) + +`feasibility.py` prepares a full pinned NodeGoat history in a private bare repository, +verifies the declared source/license blobs, and freezes 21 transactions before any +scanner invocation: three each of identical-tip, function edit, dependency edit, +configuration edit, cold, 100-module growth and multi-ref workloads. Each changed-code +repetition has distinct content. It invokes the production CLI through `trace_cli.py`, +which records engine-process durations and exit receipts without changing analysis. +Frontend, overlay, census and query costs remain separate. A 600-second priming attempt +is outside the gate population; each measured transaction retains the 30+2-second +budget. An incomplete priming run does not establish warm graph reuse. + +```sh +docker run --rm --network none --memory 3g --entrypoint python \ + -v "$PWD:/context:ro" \ + -v "$HOME/.cache/openultrasast/repos:/cache:ro" \ + -v /absolute/writable/results:/results \ + openultrasast:dev /context/benchmarks/push/feasibility.py \ + --manifest /context/benchmarks/push/inputs.json --cache /cache --out /results/new-feasibility +``` + +The output directory must not exist. `profile.json` records the planned commits, +source byte receipt, installed semantics, CPU information/affinity and container memory +limit. Every run retains the full artifact, stdout/stderr, process-stage receipts and +measurement. `scorecard.json` retains missing runs and timeouts in the declared population, +reports nearest-rank p50/p95 with sample counts, and never treats no answers as complete. +Exit 1 with a complete scorecard is measured NO-GO; a missing scorecard is an instrument +failure. Three samples per workload on development NodeGoat cannot independently qualify +a production language/framework/family or establish broad multi-project performance. diff --git a/benchmarks/push/feasibility.py b/benchmarks/push/feasibility.py new file mode 100644 index 0000000..b625f13 --- /dev/null +++ b/benchmarks/push/feasibility.py @@ -0,0 +1,272 @@ +"""Frozen repeated NodeGoat runtime experiment; never a capability qualification.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import signal +import subprocess +import sys +import time +from pathlib import Path + + +def summarize(plan, runs): + if not plan: + raise ValueError("empty measurement population") + ids = {row["id"] for row in plan} + observed = {row["id"]: row for row in runs} + if len(ids) != len(plan) or len(observed) != len(runs) or not observed.keys() <= ids: + raise ValueError("duplicate or unplanned measurement") + for row in runs: + elapsed = row.get("elapsed_seconds") + if elapsed is not None and (type(elapsed) not in (int, float) or not math.isfinite(elapsed) or elapsed < 0): + raise ValueError("invalid elapsed measurement") + result = { + "workloads": {}, + "runtime_verdict": "PASS", + "rollout": "NO-GO", + "qualification_limit": "Development NodeGoat, n=3 per workload; independent quality and multi-project performance not established.", + } + for name in dict.fromkeys(row["workload"] for row in plan): + population = [p for p in plan if p["workload"] == name] + records = [observed[p["id"]] for p in population if p["id"] in observed] + samples = sorted(r["elapsed_seconds"] for r in records if r.get("elapsed_seconds") is not None) + complete = sum(bool(r.get("target_completed")) for r in records) + target_count = sum(r.get("completed_targets", int(bool(r.get("target_completed")))) for r in records) + expected_count = sum(p.get("expected_targets", 1) for p in population) + covered = sum(r.get("coverage") == "complete_within_scope" and not r.get("deferred") for r in records) + row = dict( + population=len(population), + observed=len(records), + missing=len(population) - len(records), + latency_samples=len(samples), + p50_seconds=samples[math.ceil(0.5 * len(samples)) - 1] if samples else None, + p95_seconds=samples[math.ceil(0.95 * len(samples)) - 1] if samples else None, + timeouts=sum(bool(r.get("timed_out")) for r in records), + completed_targets=target_count, + expected_targets=expected_count, + complete_target_cases=complete, + completion=complete / len(population), + complete_coverage_cases=covered, + cancellation_overrun_seconds=max( + [max(0, r["elapsed_seconds"] - 30) for r in records if r.get("timed_out") and r.get("elapsed_seconds") is not None] or [0] + ), + ) + row["pass"] = ( + not row["missing"] + and len(samples) == len(population) + and complete / len(population) >= 0.95 + and covered / len(population) >= 0.95 + and max(samples, default=33) <= 32 + and (name == "cold" or row["p95_seconds"] <= 30) + and row["cancellation_overrun_seconds"] <= 2 + and not any(r.get("instrument_error") for r in records) + ) + if not row["pass"]: + result["runtime_verdict"] = "NO-GO" + result["workloads"][name] = row + required = {"identical-tip", "function-edit", "dependency-edit", "configuration-edit", "cold", "growth", "multi-ref"} + result["missing_workloads"] = sorted(required - result["workloads"].keys()) + if result["missing_workloads"]: + result["runtime_verdict"] = "NO-GO" + return result + + +def write(path, data): + path.write_text(json.dumps(data, indent=2, allow_nan=False) + "\n") + + +def main(): + from measure import git, pin_snapshot + + from openultrasast.model.scan import ScanBudget + from openultrasast.push.runner import _provenance + from openultrasast.repos import checkout_path, load_repo_recipe + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=False) + out = args.out.resolve() + manifest = json.loads(args.manifest.read_bytes()) + snapshots = {s["id"]: s for s in manifest["snapshots"]} + selected = [s for s in snapshots if s in ("nodegoat-vulnerable", "nodegoat-fixed")] + if len(selected) != 2: + raise ValueError("declared NodeGoat pair unavailable") + recipe = load_repo_recipe(args.manifest.parent / snapshots[selected[0]]["recipe"]) + source = checkout_path(recipe, args.cache) + bare = out / "objects.git" + bare.mkdir() + git(bare, "init", "--bare") + objects = git(source, "rev-parse", "--path-format=absolute", "--git-path", "objects").decode().strip() + (bare / "objects/info/alternates").write_text(objects + "\n") + pins = {sid: pin_snapshot(bare, recipe.commit, snapshots[sid]) for sid in selected} + head, base = pins["nodegoat-vulnerable"], pins["nodegoat-fixed"] + target_path = "app/routes/contributions.js" + original = git(bare, "cat-file", "blob", head + ":" + target_path) + assert len(original) > 0 + + def mutate(changes, title): + git(bare, "read-tree", head) + for path, data in changes.items(): + blob = git(bare, "hash-object", "-w", "--stdin", data=data).decode().strip() + git(bare, "update-index", "--add", "--cacheinfo", "100644", blob, path) + tree = git(bare, "write-tree").decode().strip() + return git(bare, "commit-tree", tree, "-p", head, data=(title + "\n").encode()).decode().strip() + + package = json.loads(git(bare, "cat-file", "blob", head + ":package.json")) + configuration = git(bare, "cat-file", "blob", head + ":config/env/all.js") + assert b"4000" in configuration and original.count(b"eval(req.body.preTax)") == 1 + plan = [] + for repeat in range(1, 4): + edited = mutate( + {target_path: original.replace(b"eval(req.body.preTax)", f"Number(req.body.preTax) + {repeat}".encode(), 1)}, + f"function edit {repeat}", + ) + dependency = dict(package) + dependency["dependencies"] = dict(package["dependencies"]) + dependency["dependencies"]["express"] = f"4.22.{repeat}" + dep = mutate({"package.json": (json.dumps(dependency, indent=2) + "\n").encode()}, f"dependency edit {repeat}") + config = mutate({"config/env/all.js": configuration.replace(b"4000", str(4100 + repeat).encode(), 1)}, f"config edit {repeat}") + growth = mutate( + { + f"app/routes/runtime-growth-{i}.js": ( + f"module.exports = function route{i}(req, res) {{ return res.json({{value: req.body.value, revision: {repeat}}}); }};\n" + ).encode() + for i in range(100) + }, + f"growth {repeat}", + ) + for workload, comparisons in [ + ("identical-tip", [(base, head)]), + ("function-edit", [(head, edited)]), + ("dependency-edit", [(head, dep)]), + ("configuration-edit", [(head, config)]), + ("cold", [(base, head)]), + ("growth", [(head, growth)]), + ("multi-ref", [(head, edited), (head, config)]), + ]: + plan.append(dict(id=f"{workload}-{repeat}", workload=workload, comparisons=comparisons, expected_targets=len(comparisons))) + installed = _provenance(bare, ScanBudget(max_model_calls=0, max_regions=500, order_by_evidence=True)) + profile = dict( + schema_version=1, + plan=plan, + source_commit=recipe.commit, + pins=pins, + input_probe=dict(path=target_path, bytes=len(original), sha256=hashlib.sha256(original).hexdigest()), + provenance=installed, + hardware=dict( + platform=platform.platform(), + cpu_affinity=sorted(os.sched_getaffinity(0)), + cpuinfo=Path("/proc/cpuinfo").read_text(), + memory_limit=Path("/sys/fs/cgroup/memory.max").read_text().strip(), + ), + gates=dict(deadline_seconds=30, cancellation_seconds=2, warm_p95_seconds=30, completion=0.95), + lab_priming_deadline=600, + quantile="nearest-rank; includes timeout elapsed values; missing samples explicit", + target=dict(path=target_path, function="handleContributionsUpdate", family="injection"), + harness_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + instrumentation_sha256=hashlib.sha256(Path(__file__).with_name("trace_cli.py").read_bytes()).hexdigest(), + ) + write(out / "profile.json", profile) + runs = [] + + def run(case, deadline, cache=True): + identity = case["id"] + artifact = out / (identity + ".json") + trace = out / (identity + ".trace.jsonl") + lines = "".join(f"refs/heads/local{i} {tip} refs/heads/result{i} {old}\n" for i, (old, tip) in enumerate(case["comparisons"])) + command = [ + sys.executable, + str(Path(__file__).with_name("trace_cli.py")), + "pre-push", + str(bare), + "--remote", + "evaluation", + "/local-only", + "--artifact", + str(artifact), + "--deadline", + str(deadline), + ] + if cache: + command += ["--cache-dir", str(out / "artifacts")] + start = time.monotonic() + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + env={**os.environ, "OUSAST_EVALUATION_TRACE": str(trace)}, + ) + error = None + try: + stdout, stderr = process.communicate(lines, timeout=deadline + 5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate(timeout=2) + error = "outer_deadline_exceeded" + elapsed = time.monotonic() - start + (out / (identity + ".stdout")).write_text(stdout) + (out / (identity + ".stderr")).write_text(stderr) + data = json.loads(artifact.read_bytes()) if artifact.is_file() else None + if data is None: + error = error or "missing_artifact" + if process.returncode != 0: + error = error or "unexpected_cli_exit" + completed = 0 + if data: + for scan in data["scans"]: + if scan["side"] == "head" and any( + q["status"] == "completed" and all(q["identity"][key] == value for key, value in profile["target"].items()) + for q in scan["scan"]["question_outcomes"] + ): + completed += 1 + record = dict( + id=identity, + elapsed_seconds=elapsed, + exit_code=process.returncode, + instrument_error=error, + timed_out=bool(data and any("deadline" in reason for reason in data["admission"]["coverage_reasons"])) + or error == "outer_deadline_exceeded", + target_completed=completed == case["expected_targets"], + completed_targets=completed, + expected_targets=case["expected_targets"], + coverage=data["result"]["coverage_status"] if data else "unavailable", + timings=data["timings"] if data else {}, + selected=sum(len(s["scan"]["scope"]["selected"]) for s in data["scans"] if s["side"] == "head" and s["scan"]["scope"]) + if data + else 0, + deferred=sum(len(s["scan"]["scope"]["deferred"]) for s in data["scans"] if s["side"] == "head" and s["scan"]["scope"]) + if data + else 0, + stages=[json.loads(line) for line in trace.read_text().splitlines()] if trace.exists() else [], + ) + write(out / (identity + ".measurement.json"), record) + print(json.dumps({k: record[k] for k in ("id", "elapsed_seconds", "target_completed", "coverage", "instrument_error")}), flush=True) + return record + + prime = run(dict(id="lab-prime", comparisons=[(base, head)], expected_targets=1), 600) + write(out / "priming.json", prime) + for case in plan: + runs.append(run(case, 30, cache=case["workload"] != "cold")) + write(out / "runs.json", runs) + write(out / "scorecard.json", summarize(plan, runs)) + final = summarize(plan, runs) + final["profile_sha256"] = hashlib.sha256((out / "profile.json").read_bytes()).hexdigest() + final["priming_completed"] = prime["target_completed"] + write(out / "scorecard.json", final) + return 0 if final["runtime_verdict"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/push/trace_cli.py b/benchmarks/push/trace_cli.py new file mode 100644 index 0000000..94b68bf --- /dev/null +++ b/benchmarks/push/trace_cli.py @@ -0,0 +1,53 @@ +"""Transparent engine-process timing around the production CLI (evaluation only).""" + +import json +import os +import sys +import time +from pathlib import Path + +from openultrasast.cli import main +from openultrasast.cpg.backend import JoernBackend + +original = JoernBackend._run_bounded +trace = Path(os.environ["OUSAST_EVALUATION_TRACE"]) + + +def measured(self, command, **kwargs): + executable = Path(command[0]).name + scripts = [Path(arg).name for arg in command if arg.endswith(".sc")] + stage = ( + "overlay" + if "overlay.sc" in scripts + else "census" + if "census.sc" in scripts + else "query" + if scripts + else "frontend" + if "cpg" in executable or "joern-parse" in executable + else "input_probe" + ) + started = time.monotonic() + result = None + try: + result = original(self, command, **kwargs) + return result + finally: + with trace.open("a") as stream: + stream.write( + json.dumps( + { + "stage": stage, + "seconds": time.monotonic() - started, + "exit_code": result.returncode if result is not None else None, + "receipt": result is not None, + "executable": executable, + "scripts": scripts, + } + ) + + "\n" + ) + + +JoernBackend._run_bounded = measured +raise SystemExit(main(sys.argv[1:])) diff --git a/tests/test_push_feasibility.py b/tests/test_push_feasibility.py new file mode 100644 index 0000000..96c110a --- /dev/null +++ b/tests/test_push_feasibility.py @@ -0,0 +1,51 @@ +import importlib.util +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location("feasibility", Path("benchmarks/push/feasibility.py")) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def test_timeouts_and_missing_runs_remain_in_fixed_population(): + plan = [{"id": str(i), "workload": "function-edit"} for i in range(3)] + runs = [ + {"id": "0", "elapsed_seconds": 2, "target_completed": True, "timed_out": False}, + {"id": "1", "elapsed_seconds": 30.5, "target_completed": False, "timed_out": True}, + ] + result = module.summarize(plan, runs) + row = result["workloads"]["function-edit"] + assert row["population"] == 3 and row["missing"] == 1 + assert row["p95_seconds"] == 30.5 and row["timeouts"] == 1 + assert row["completed_targets"] == 1 + assert result["runtime_verdict"] == "NO-GO" + + +def test_duplicate_or_unplanned_measurement_is_rejected(): + plan = [{"id": "a", "workload": "cold"}] + with pytest.raises(ValueError): + module.summarize(plan, [{"id": "outside"}]) + with pytest.raises(ValueError): + module.summarize(plan, [{"id": "a"}, {"id": "a"}]) + + +def test_identical_tip_alone_cannot_qualify_runtime(): + plan = [{"id": "a", "workload": "identical-tip"}] + runs = [{"id": "a", "elapsed_seconds": 1, "target_completed": True, "coverage": "complete_within_scope", "deferred": 0}] + assert module.summarize(plan, runs)["runtime_verdict"] == "NO-GO" + + +@pytest.mark.parametrize("elapsed", [-1, float("nan"), float("inf")]) +def test_invalid_elapsed_is_an_instrument_error(elapsed): + with pytest.raises(ValueError): + module.summarize([{"id": "a", "workload": "cold"}], [{"id": "a", "elapsed_seconds": elapsed}]) + + +def test_multi_ref_target_denominator_is_fixed_before_execution(): + result = module.summarize( + [{"id": "a", "workload": "multi-ref", "expected_targets": 2}], + [{"id": "a", "elapsed_seconds": 30.5, "target_completed": False, "completed_targets": 1, "timed_out": True}], + ) + assert result["workloads"]["multi-ref"]["expected_targets"] == 2 + assert result["workloads"]["multi-ref"]["completed_targets"] == 1