From e07ab2ac932d2ffbc26f80aefe7baf2bd592e785 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:29:51 +0200 Subject: [PATCH] measure: 5 paired repeats, DSPy hunt baseline 30% vs pilot hunter 52% recall on the 12 positives Co-Authored-By: Claude Opus 5.5 --- .../2026-10-13-hunt-repeats/NOTES.md | 79 ++++++++++++ .../2026-10-13-hunt-repeats/analysis.json | 122 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 benchmarks/measurements/2026-10-13-hunt-repeats/NOTES.md create mode 100644 benchmarks/measurements/2026-10-13-hunt-repeats/analysis.json diff --git a/benchmarks/measurements/2026-10-13-hunt-repeats/NOTES.md b/benchmarks/measurements/2026-10-13-hunt-repeats/NOTES.md new file mode 100644 index 0000000..8000bf0 --- /dev/null +++ b/benchmarks/measurements/2026-10-13-hunt-repeats/NOTES.md @@ -0,0 +1,79 @@ +# Hunt baseline against the pilot hunter: 5 paired repeats on the 12 pilot positives (2026-10-13) + +## The question + +On the first single runs, the baseline DSPy hunt program found 3 of the 12 pilot positives and the pilot hunter +found 6. One run per arm cannot separate the arms, because the provider's answers vary even at temperature 0. +These five repeats measure the gap. + +## Setup + +- **Changes:** the 12 fix-site positives of the valid pilot (`benchmarks/valid/pilot/pool-valid.toml`, `--only vulnerability`). +- **Arm A, the pilot hunter:** `benchmarks/valid/hunter_arm.py`, classifier on, exactly as it ran in the pilot. +- **Arm B, the DSPy baseline:** `python -m openultrasast.hunt.cli run`, with the baseline instruction, all annotations, ReAct over the three + repository tools and `--max-iters 8`. Each run had its own empty response cache, so no run replayed another. +- **Runs:** five repeats per arm, paired by change, run one after another on 2026-10-13 from 13:20 to 15:27. +- **Analysis:** `analyse.py` was committed before any result existed (26720ed) and was not changed afterwards. + - The unit is one change in one repeat (60 per arm). + - A hit uses `benchmarks.unseen.score.catches`. + - Failed and unparsed hunts stay in the denominator as misses. + +## Results + +| | Pilot hunter (A) | DSPy baseline (B) | +|---|---|---| +| Recall | 31/60 = 52% (95% Wilson 39-64%) | 18/60 = 30% (95% Wilson 20-43%) | +| Hunts with an unparseable answer | 0 | 7 of 60 (12%) | +| Instrument failures | 0 | 0 | +| Suspicions raised per change | 5.8 | 0.6 | + +- **Paired difference, B minus A:** -0.22. The 95% interval, from a bootstrap that resamples whole changes (5000 resamples), is -0.38 to -0.05. +- **Exact McNemar:** 14 units were caught only by the hunter and 1 only by DSPy, with 45 concordant (p = 0.001). + +The pilot hunter is ahead, and the gap does not come from run-to-run noise. The DSPy baseline is the starting point for +the learning loop, not a candidate to ship, so this is the bar the loop has to clear. + +Per-change hit rate over the five repeats (hunter / DSPy): + +| Repository | Hunter | DSPy | +|---|---|---| +| imzbf/md-editor-v3 | 1.0 | 1.0 | +| zenitram/malla | 1.0 | 0.8 | +| nvbn/thefuck | 0.8 | 1.0 | +| jupyter-lsp/jupyterlab-lsp | 1.0 | 0.4 | +| ondata/ckan-mcp-server | 1.0 | 0.2 | +| fastapi-users/fastapi-users | 0.6 | 0.0 | +| webpack/webpack-bundle-analyzer | 0.4 | 0.0 | +| contentful/contentful-mcp-server | 0.2 | 0.2 | +| i18next/i18next-http-middleware | 0.2 | 0.0 | +| bytebase/dbhub | 0.0 | 0.0 | +| matrix-org/matrix-appservice-bridge | 0.0 | 0.0 | +| zereight/gitlab-mcp | 0.0 | 0.0 | + +Neither arm ever caught three of the changes (dbhub, matrix-appservice-bridge, gitlab-mcp). + +## Where DSPy loses + +- **It is too cautious.** It raises about a tenth as many suspicions as the hunter. Its misses are mostly + empty answers, not wrong ones. +- **The tool loop sometimes breaks.** 7 of its 60 hunts (fe1cccd1 three times, then 415e1c7c, a318f3e3, a70b5501 and + 090e3500 once each) failed because of format, not search. The raw responses in the per-run caches show the cause: + - On one step, the model copies the headings DSPy uses for earlier steps. It writes `tool_name` instead of + `next_tool_name`, or writes a whole fake history (`thought_1` .. `thought_42`). + - DSPy then retries the step in JSON. When that retry also fails, DSPy 3.4.1 raises `AdapterParseError`. + - The tool loop only stops early on a `ValueError` (`dspy/predict/react.py:104`), and `AdapterParseError` is not a + `ValueError`. So the error escapes the loop, the findings step never runs, and the hunt is recorded as empty. + + Even with all 7 counted as hits, DSPy would be at 25/60. The gap to the hunter is mostly search, not format. + +## Cost + +- **DSPy arm:** 17.3M prompt tokens (94.8% served from the provider's prompt cache) and 0.94M completion tokens over the five runs. That is + $1.89 at the registered deepseek-flash prices. +- **Pilot hunter arm:** these records do not carry its token usage, so its cost is not stated here. + +## Files + +- `analyse.py`: the pre-registered analysis. +- `analysis.json`: its output on the five repeats. +- Raw runs (records, logs, per-run caches): `~/ousast-results/hunt/repeats/{dspy,hunter}-{1..5}`. diff --git a/benchmarks/measurements/2026-10-13-hunt-repeats/analysis.json b/benchmarks/measurements/2026-10-13-hunt-repeats/analysis.json new file mode 100644 index 0000000..668bd5a --- /dev/null +++ b/benchmarks/measurements/2026-10-13-hunt-repeats/analysis.json @@ -0,0 +1,122 @@ +{ + "units": 60, + "repeats": [ + 1, + 2, + 3, + 4, + 5 + ], + "hunter": { + "hits": 31, + "units": 60, + "recall": 0.5166666666666667, + "wilson95": [ + 0.39307810032592494, + 0.6382495046026722 + ], + "instrument_failures": 0, + "parse_failures": 0, + "suspicions_mean": 5.766666666666667 + }, + "dspy": { + "hits": 18, + "units": 60, + "recall": 0.3, + "wilson95": [ + 0.19898166138769108, + 0.42508707946914487 + ], + "instrument_failures": 0, + "parse_failures": 7, + "suspicions_mean": 0.6333333333333333 + }, + "dspy_minus_hunter": { + "diff": -0.21666666666666673, + "ci95": [ + -0.3833333333333334, + -0.050000000000000044 + ], + "changes": 12 + }, + "mcnemar": { + "only_hunter": 14, + "only_dspy": 1, + "concordant": 45, + "p": 0.0009765625 + }, + "per_change": [ + { + "change": "a70b5501fedbfaf33aed49d3edc8534353aa6b05adccbb36019170b7f99fcdfe", + "repo": "bytebase/dbhub", + "dspy": 0.0, + "hunter": 0.0 + }, + { + "change": "090e3500634d126cae6d9fccb5902647295875403e69bdb0389c86b4fa3c0086", + "repo": "contentful/contentful-mcp-server", + "dspy": 0.2, + "hunter": 0.2 + }, + { + "change": "6d3b0736e6d91fc5f182954e2e19565dfb328931fb85952695a5895eb0471a9d", + "repo": "fastapi-users/fastapi-users", + "dspy": 0.0, + "hunter": 0.6 + }, + { + "change": "c7978d72ecdfaa6cf9c71f6a9baea3f6a64d922b25ab78279e3fc116c25b0c4b", + "repo": "i18next/i18next-http-middleware", + "dspy": 0.0, + "hunter": 0.2 + }, + { + "change": "1ad15598b335ce9ebfca883925fa2dcd5031908c8dfbba5f5e0f4d86794255af", + "repo": "imzbf/md-editor-v3", + "dspy": 1.0, + "hunter": 1.0 + }, + { + "change": "415e1c7c5676cba1813e6c0a26ef89ead75e579d4c3ed27ebb1831865c6c8981", + "repo": "jupyter-lsp/jupyterlab-lsp", + "dspy": 0.4, + "hunter": 1.0 + }, + { + "change": "59611a8c4ee8d9d6e12b9e38b79ea374fe8a33657d51e262a3953327d180aaae", + "repo": "matrix-org/matrix-appservice-bridge", + "dspy": 0.0, + "hunter": 0.0 + }, + { + "change": "545fcc9756f7ea72eaa4971fd8bc654cf8c7bacf519388057ad8c5236ff656fa", + "repo": "nvbn/thefuck", + "dspy": 1.0, + "hunter": 0.8 + }, + { + "change": "a318f3e3f4f42ec2ba58bec37fe98d50baf5cd7ad8d5e30fc96b880b930114eb", + "repo": "ondata/ckan-mcp-server", + "dspy": 0.2, + "hunter": 1.0 + }, + { + "change": "cc735ccb90a5cdbe8f59f2f9336261efffaae75aa115e09d2a15ca5e3ddca894", + "repo": "webpack/webpack-bundle-analyzer", + "dspy": 0.0, + "hunter": 0.4 + }, + { + "change": "8cb82ba93a6a146417537fdebbd7c23a97513cbc9fa8f7457309b3823375bf7f", + "repo": "zenitram/malla", + "dspy": 0.8, + "hunter": 1.0 + }, + { + "change": "fe1cccd1401bd6412f5f4d8559f6498156a9813a3ba6bca83d1c84c669c40e70", + "repo": "zereight/gitlab-mcp", + "dspy": 0.0, + "hunter": 0.0 + } + ] +}