mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5
5e2070bc17
Twelve injection pairs, both sides, K=5, temperature 0, thinking disabled, recording every tool call. $0.23. The statistic I designed for could not be computed: across 120 runs there was not one repeated trajectory -- 4.67 and 4.92 distinct trajectories per side out of 5, and 78 distinct tool-name sequences ignoring arguments entirely. The detector is not sampling one measurement five times; it is running five different investigations and voting on the results. As a gradient the relationship is monotone: runs whose trajectories were similar (>=0.75 Jaccard) disagreed 0% of the time, mid-similarity 4.2%, low-similarity 30.9%. Three more facts. The search is grep: 555 grep_repo calls against 181 read_file, 71 entry_points, 43 find_refs, 13 read_definition -- the model is guessing patterns to locate sinks the CST and the entry-point mapper already know. All the instability is on the vulnerable side; every pair produced exactly one finding set on the fixed side, so the noise is in detection and none of it in false positives. And findings converge much more than trajectories do: 78 trajectories, one to three finding sets per pair. The floor is therefore not measuring uncertainty about a question. It is measuring how often an unconstrained search happens to reach the sink. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
155 lines
4.3 KiB
JSON
155 lines
4.3 KiB
JSON
{
|
|
"agreement_by_trajectory_similarity": {
|
|
"high>=0.75": {
|
|
"disagreed": 0,
|
|
"rate": 0.0,
|
|
"run_pairs": 2
|
|
},
|
|
"low<0.4": {
|
|
"disagreed": 29,
|
|
"rate": 0.30851063829787234,
|
|
"run_pairs": 94
|
|
},
|
|
"mid0.4-0.75": {
|
|
"disagreed": 1,
|
|
"rate": 0.041666666666666664,
|
|
"run_pairs": 24
|
|
}
|
|
},
|
|
"command": "scratchpad/trajectory-experiment.py 12 (injection family, vibe-py, K=5, both sides)",
|
|
"cost_usd": 0.23402034400000005,
|
|
"distinct_tool_name_sequences": 78,
|
|
"generated": "2026-09-06",
|
|
"identical_trajectory_run_pairs": 0,
|
|
"k_runs": 5,
|
|
"model": "deepseek-v4-flash",
|
|
"note": "Is the run-to-run disagreement a matter of judgement or of the loop's freedom? Same pair, same prompt, temperature 0, five runs, recording the tool-call sequence each run took. The intended statistic \u2014 P(disagree | identical trajectory) \u2014 could not be computed, because across 120 runs there was not a single repeated trajectory. Asked as a gradient instead, outcome agreement tracks trajectory similarity monotonically. The detector is not sampling one measurement five times; it is running five different investigations, and the noise floor measures how often an unconstrained search happens to reach the sink.",
|
|
"pairs": 12,
|
|
"pairs_detail": [
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 2,
|
|
"outcomes": {
|
|
"both_silent": 4,
|
|
"pair_correct": 1
|
|
},
|
|
"pair": "damn-vulnerable-graphql-application-views-mutate"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 1,
|
|
"outcomes": {
|
|
"pair_correct": 5
|
|
},
|
|
"pair": "damn-vulnerable-graphql-application-views-resolve-pastes"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 1,
|
|
"outcomes": {
|
|
"pair_correct": 5
|
|
},
|
|
"pair": "damn-vulnerable-graphql-application-views-resolve-system-dia"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 1,
|
|
"outcomes": {
|
|
"pair_correct": 5
|
|
},
|
|
"pair": "djangoat-analytics-hits-by-ip"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 1,
|
|
"outcomes": {
|
|
"both_silent": 5
|
|
},
|
|
"pair": "djangoat-views-user-view"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 2,
|
|
"outcomes": {
|
|
"both_silent": 3,
|
|
"pair_correct": 2
|
|
},
|
|
"pair": "dsvpwa-attacks-run"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 2,
|
|
"outcomes": {
|
|
"both_silent": 3,
|
|
"pair_correct": 2
|
|
},
|
|
"pair": "dsvpwa-handlers-do-bdr"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 3,
|
|
"outcomes": {
|
|
"both_silent": 1,
|
|
"pair_correct": 4
|
|
},
|
|
"pair": "dvpwa-student-create"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 2,
|
|
"outcomes": {
|
|
"both_silent": 2,
|
|
"pair_correct": 3
|
|
},
|
|
"pair": "extremely-vulnerable-flask-app-account-search"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 2,
|
|
"outcomes": {
|
|
"both_silent": 4,
|
|
"pair_correct": 1
|
|
},
|
|
"pair": "extremely-vulnerable-flask-app-signup-validate-token"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 2,
|
|
"outcomes": {
|
|
"pair_correct": 5
|
|
},
|
|
"pair": "threatbyte-routes-dashboard"
|
|
},
|
|
{
|
|
"distinct_fixed_findings": 1,
|
|
"distinct_trajectories": 5,
|
|
"distinct_vuln_findings": 1,
|
|
"outcomes": {
|
|
"pair_correct": 5
|
|
},
|
|
"pair": "vampi-user-model-get-user"
|
|
}
|
|
],
|
|
"runs_recorded": 120,
|
|
"temperature": 0,
|
|
"thinking": false,
|
|
"tool_calls": {
|
|
"entry_points": 71,
|
|
"find_refs": 43,
|
|
"grep_repo": 555,
|
|
"read_definition": 13,
|
|
"read_file": 181
|
|
}
|
|
}
|