From 5e2070bc17619a9fa55c8c502fbf413a0b1c42eb Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:49:54 +0200 Subject: [PATCH] measure(detector-optimizer): the detector never runs the same investigation twice Twelve injection pairs, both sides, K=5, temperature 0, thinking disabled, recording every tool call. $0.23. The statistic I designed for could not be computed: across 120 runs there was not one repeated trajectory -- 4.67 and 4.92 distinct trajectories per side out of 5, and 78 distinct tool-name sequences ignoring arguments entirely. The detector is not sampling one measurement five times; it is running five different investigations and voting on the results. As a gradient the relationship is monotone: runs whose trajectories were similar (>=0.75 Jaccard) disagreed 0% of the time, mid-similarity 4.2%, low-similarity 30.9%. Three more facts. The search is grep: 555 grep_repo calls against 181 read_file, 71 entry_points, 43 find_refs, 13 read_definition -- the model is guessing patterns to locate sinks the CST and the entry-point mapper already know. All the instability is on the vulnerable side; every pair produced exactly one finding set on the fixed side, so the noise is in detection and none of it in false positives. And findings converge much more than trajectories do: 78 trajectories, one to three finding sets per pair. The floor is therefore not measuring uncertainty about a question. It is measuring how often an unconstrained search happens to reach the sink. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/detector-optimizer/brief.md | 42 ++++- .../2026-09-06-trajectory-divergence.json | 154 ++++++++++++++++++ 2 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 benchmarks/measurements/2026-09-06-trajectory-divergence.json diff --git a/.kiro/specs/detector-optimizer/brief.md b/.kiro/specs/detector-optimizer/brief.md index 9416fd3..81b5f42 100644 --- a/.kiro/specs/detector-optimizer/brief.md +++ b/.kiro/specs/detector-optimizer/brief.md @@ -173,11 +173,43 @@ fails on an under-specified task and does well on a narrow typed one. So the che the more of the return sits in decomposition and constraints and the less in prompt evolution — which is the opposite of the order this spec originally proposed. -**The experiment that settles it, before any of this is built.** We already run K = 5. Record the tool-call -sequence for every run, and measure how often a pair's runs took different trajectories, and whether -outcome flips coincide with trajectory divergence. If they do, the noise architecture is treating a -structural problem as a statistical one and most of it can be deleted rather than repaired. Cost: nothing -beyond logging what we already generate. +### The experiment, run (2026-09-06, `benchmarks/measurements/2026-09-06-trajectory-divergence.json`) + +Twelve injection pairs, both sides, K = 5, temperature 0, thinking disabled, recording the tool-call +sequence of every run. $0.23. + +**The intended statistic could not be computed. Across 120 runs there was not one repeated trajectory** — +4.67 distinct trajectories per vulnerable side and 4.92 per fixed side, out of 5, and 78 distinct tool-*name* +sequences ignoring arguments entirely. The detector is not sampling one measurement five times. It is running +five different investigations and voting on the results. + +Asked as a gradient instead, the relationship is monotone and clean: + +| trajectory similarity (Jaccard over tool calls) | run pairs | disagreed on the outcome | +|---|---|---| +| high (≥ 0.75) | 2 | 0 (0.0%) | +| mid (0.4–0.75) | 24 | 1 (4.2%) | +| low (< 0.4) | 94 | 29 (30.9%) | + +Three more facts from the same run: + +- **The variance is a search problem, and the search is `grep_repo`.** 555 grep calls against 181 `read_file`, + 71 `entry_points`, 43 `find_refs`, 13 `read_definition`. The model spends its budget guessing patterns to + locate a sink — while the CST, the entry-point mapper, `flows` and `obligations` already know where the + sinks are. +- **All of the instability is on the vulnerable side.** Every one of the twelve pairs produced exactly one + distinct finding set on the fixed side. The detector is reliably silent on the fix and unreliably vocal on + the bug, so the noise is entirely in detection and none of it in false positives. +- **Findings converge far more than trajectories do.** 78 distinct trajectories, but one to three distinct + finding sets per pair. Many different investigations reach the same answer; six of twelve pairs are stable + despite never once repeating a trajectory. + +**What this does to the noise architecture.** The floor is not measuring the detector's uncertainty about a +question. It is measuring how often an unconstrained search happens to reach the sink. A majority vote over +five runs is therefore not variance reduction — it is an unweighted ensemble of five arbitrary search +procedures — and a regression budget derived from it is a budget for the search's luck. K runs, the floor, +the budget and the sign test are all statistical treatment for a structural problem, and the 0% and 4.2% +rows of that table say what happens when the structure is fixed instead. ## The constraint that outranks the algorithm diff --git a/benchmarks/measurements/2026-09-06-trajectory-divergence.json b/benchmarks/measurements/2026-09-06-trajectory-divergence.json new file mode 100644 index 0000000..370be8d --- /dev/null +++ b/benchmarks/measurements/2026-09-06-trajectory-divergence.json @@ -0,0 +1,154 @@ +{ + "agreement_by_trajectory_similarity": { + "high>=0.75": { + "disagreed": 0, + "rate": 0.0, + "run_pairs": 2 + }, + "low<0.4": { + "disagreed": 29, + "rate": 0.30851063829787234, + "run_pairs": 94 + }, + "mid0.4-0.75": { + "disagreed": 1, + "rate": 0.041666666666666664, + "run_pairs": 24 + } + }, + "command": "scratchpad/trajectory-experiment.py 12 (injection family, vibe-py, K=5, both sides)", + "cost_usd": 0.23402034400000005, + "distinct_tool_name_sequences": 78, + "generated": "2026-09-06", + "identical_trajectory_run_pairs": 0, + "k_runs": 5, + "model": "deepseek-v4-flash", + "note": "Is the run-to-run disagreement a matter of judgement or of the loop's freedom? Same pair, same prompt, temperature 0, five runs, recording the tool-call sequence each run took. The intended statistic \u2014 P(disagree | identical trajectory) \u2014 could not be computed, because across 120 runs there was not a single repeated trajectory. Asked as a gradient instead, outcome agreement tracks trajectory similarity monotonically. The detector is not sampling one measurement five times; it is running five different investigations, and the noise floor measures how often an unconstrained search happens to reach the sink.", + "pairs": 12, + "pairs_detail": [ + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 2, + "outcomes": { + "both_silent": 4, + "pair_correct": 1 + }, + "pair": "damn-vulnerable-graphql-application-views-mutate" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 1, + "outcomes": { + "pair_correct": 5 + }, + "pair": "damn-vulnerable-graphql-application-views-resolve-pastes" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 1, + "outcomes": { + "pair_correct": 5 + }, + "pair": "damn-vulnerable-graphql-application-views-resolve-system-dia" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 1, + "outcomes": { + "pair_correct": 5 + }, + "pair": "djangoat-analytics-hits-by-ip" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 1, + "outcomes": { + "both_silent": 5 + }, + "pair": "djangoat-views-user-view" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 2, + "outcomes": { + "both_silent": 3, + "pair_correct": 2 + }, + "pair": "dsvpwa-attacks-run" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 2, + "outcomes": { + "both_silent": 3, + "pair_correct": 2 + }, + "pair": "dsvpwa-handlers-do-bdr" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 3, + "outcomes": { + "both_silent": 1, + "pair_correct": 4 + }, + "pair": "dvpwa-student-create" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 2, + "outcomes": { + "both_silent": 2, + "pair_correct": 3 + }, + "pair": "extremely-vulnerable-flask-app-account-search" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 2, + "outcomes": { + "both_silent": 4, + "pair_correct": 1 + }, + "pair": "extremely-vulnerable-flask-app-signup-validate-token" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 2, + "outcomes": { + "pair_correct": 5 + }, + "pair": "threatbyte-routes-dashboard" + }, + { + "distinct_fixed_findings": 1, + "distinct_trajectories": 5, + "distinct_vuln_findings": 1, + "outcomes": { + "pair_correct": 5 + }, + "pair": "vampi-user-model-get-user" + } + ], + "runs_recorded": 120, + "temperature": 0, + "thinking": false, + "tool_calls": { + "entry_points": 71, + "find_refs": 43, + "grep_repo": 555, + "read_definition": 13, + "read_file": 181 + } +}