diff --git a/.kiro/specs/pre-push-safety-net/strategy-review-2026-09-28.md b/.kiro/specs/pre-push-safety-net/strategy-review-2026-09-28.md index c289173..82ec58b 100644 --- a/.kiro/specs/pre-push-safety-net/strategy-review-2026-09-28.md +++ b/.kiro/specs/pre-push-safety-net/strategy-review-2026-09-28.md @@ -82,3 +82,39 @@ declared sites, which would leak the answer). 1. Direction: run the hunter experiment first (recommended), or choose A, B or C now. 2. For the experiment: the scope handed to the model per pin, the model, and the cost ceiling. 3. Whether the roadmap's M1b/M2 work (rendering, runtime) pauses until the direction is chosen. + + +## 7. Update 2026-09-29: what the experiment showed + +Recorded in `benchmarks/independent/results-v2-model-pipeline.json` (exploratory; v2 is spent for tuning). + +**Instrument first.** Four defects had made every model-backed result since 2026-09-10 unmeasured: the default +model id was the product name and the API rejected it (cf57995); the hunter reached the embeddings provider, not +the chat provider (cf57995); the final-answer turn used the provider's JSON mode and the answers were lost +(3ffb8b4); one grep match in a minified bundle overflowed the model's context (8794884). The "hunter 1/17" number +in section 5's experiment was produced through the broken answer turn and is void. + +**Scope selection, source only.** The declared vulnerable function is among the candidates for: the ranker's +top-20 entry points 1/17; the shipped sink vocabulary 2/17; a model reading every product file 15/17 ($14). The +vocabulary route (option A) is confirmed not to converge on this population. + +**Verification.** Per-file triage, one 6-step tool hunt per file, two independent passes, report only agreement: +- validation set: 16 of 20 declared sites agreed, 12 of 15 cases; the 26 pgAdmin candidates that flickered + between single runs reduce to 9 stable, 4 disputed, 13 quiet; +- full run, 7 of 17 cases before the account emptied ($23): detected 4/7 (pgAdmin, qwed, LMDeploy, + ReactPress); ContextForge and FUXA found in one pass of two; winml missed because a CORS literal is triaged + "constant" -- the config family needs its own question, not the reachability one; +- cost: $0.009-0.022 per candidate for two passes (old single-pass design: $0.017); a full two-pass run of the + population is roughly $45-50. + +**Precision is the open problem.** 196 agreed findings across 7 cases (pgAdmin alone 126 of 875 verified). Five +pgAdmin findings read against source: three true (privileged; raw `{{ data.* }}` in SQL templates, unescaped +stored `schema_res`), two false with rationales that contradict their own verdict (an ORM call the model itself +calls parameterized; a pickle from the signed session). Agreement cannot catch a mistake both passes make. The +proposed next filter is a no-tools judge over each agreed finding's rationale and code (~$0.002 per finding), +which was not run. Two-pass agreement is itself not stable between runs (ContextForge, FUXA). + +**Standing.** Recall: the model route reaches 12/15 candidate-site cases where the engine reached 0/17 and the +vocabulary 2/17; that decides between options A and B in the model's favour for recall. Precision, fixed-pin +behaviour and cost at product scale are unmeasured. No gate is met. Any qualification needs population v3 and +a pre-registered protocol that also states the judge stage, the pass rule and the config-family question. diff --git a/benchmarks/independent/results-v2-model-pipeline.json b/benchmarks/independent/results-v2-model-pipeline.json new file mode 100644 index 0000000..024ef5a --- /dev/null +++ b/benchmarks/independent/results-v2-model-pipeline.json @@ -0,0 +1,114 @@ +{ + "measurement": "model-driven pipeline on population v2 (exploratory; v2 is spent for tuning): model-classified sinks -> per-file triage -> per-file 6-step tool hunts -> two-pass agreement. Stopped by the maintainer after the account emptied; 7 of 17 cases complete.", + "analyzer": "6acd354 (hunter fixes cf57995, 3ffb8b4, 8794884; verify_batched.py 072a102)", + "spend_usd": { + "stage1_model_sinks": 14.42, + "validation_46_candidates": 1.01, + "full_run_7_cases_plus_budibase_partial": 23.36 + }, + "stage1_reach": { + "declared_site_among_candidates": [ + 15, + 17 + ], + "note": "ranker top-20: 1/17; vocabulary sink-first: 2/17" + }, + "validation_set": { + "declared_sites_agreed": [ + 16, + 20 + ], + "cases": [ + 12, + 15 + ], + "flickering_pgadmin_26": { + "triaged_out": 2, + "agreed": 9, + "disputed": 4, + "quiet": 13 + } + }, + "full_run_7_cases": { + "recall": [ + 4, + 7 + ], + "recall_wilson95": [ + 0.25, + 0.842 + ], + "detected": [ + "pgadmin-maintenance-sqli", + "qwed-sympy-eval", + "lmdeploy-media-url-ssrf", + "reactpress-findall-column-sqli" + ], + "unstable_one_pass_only": [ + "contextforge-prompt-ssti", + "fuxa-tdengine-escape-sqli" + ], + "missed": [ + "winml-cli-serve-cors (triage: constant; the reachability question does not fit config_secrets)" + ], + "agreed_findings_pooled": 196, + "pgadmin": { + "candidates": 1037, + "triaged_out": 162, + "agreed": 126, + "disputed": 238, + "usd": 8.91 + }, + "not_run": [ + "budibase-mongo-template-nosqli", + "unleash-addon-webhook-ssrf", + "medplum-external-auth-open-redirect", + "cct-studio-server-cmdi", + "wp-gopay-log-filter-sqli", + "wp-user-registration-login-redirect", + "wp-directorist-avatar-ssrf", + "wp-groundhogg-confirm-redirect", + "froxlor-admin-ip-second-order-sqli", + "sulu-smart-content-sql2-injection" + ], + "fixed_side_alerts": "not measured (no recheck run)" + }, + "precision": { + "formal_sample": "66 drawn, not adjudicated (stopped)", + "spot_check_pgadmin_5": [ + { + "site": "web/pgadmin/authenticate/mfa/utils.py:370:mfa_add", + "verdict": "False, guarded", + "why": "SQLAlchemy ORM, parameterized; the rationale says so and reports anyway" + }, + { + "site": "web/pgadmin/browser/server_groups/servers/databases/schemas/__init__.py:533:nodes", + "verdict": "True, privileged", + "why": "stored schema_res split and quoted without escaping into nsp.nspname in (...)" + }, + { + "site": "web/pgadmin/browser/server_groups/servers/databases/schemas/tables/compound_triggers/__init__.py:662:update", + "verdict": "True, privileged", + "why": "update.sql interpolates {{ data.prosrc }} raw; witness did not name the field" + }, + { + "site": "web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/utils.py:286:get_sql", + "verdict": "True, privileged", + "why": "update.sql interpolates {{data.fillfactor}} and other storage parameters raw" + }, + { + "site": "web/pgadmin/misc/cloud/biganimal/__init__.py:528:deploy_on_biganimal", + "verdict": "False, not attacker", + "why": "pickle.loads of an app-produced value in the signed Flask session" + } + ], + "note": "two-pass agreement does not catch a mistake both passes make; a no-tools judge over rationale + code is the proposed next filter" + }, + "stability": "contextforge and fuxa were agreed in the validation run and only disputed in the full run: two-pass agreement varies between runs", + "cost_per_candidate_two_passes_usd": { + "validation_set": 0.022, + "pgadmin_full": 0.0086, + "old_single_pass_design": 0.0167 + }, + "gates": "none met or measurable: 7/17 cases run, precision unadjudicated, fixed pins unchecked" +}