mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5.5
56b1717246
agree matched declared sites through benchmarks/independent/evaluate.py, which the runner image does not carry, so site matching would have been "not assessed". The generator now resolves the fix's old-side ranges (evaluate.hunks, line for line) into case.json, with the declared sites in the set and the reference's cost, and agree.matches_declared reproduces matches_v2; a parity test runs both over the recorded outputs of the 15 validation cases (200 probes, identical verdicts) and agree on those outputs returns the reference's 16 of 20 and $0.022. agree reports cost_per_candidate (plane spend) and cost_per_candidate_with_recorded_triage; Requirement 6.2 is judged on the latter (design.md). --runner-image re-pins the templates from ops/ax/up.sh's pin file; the committed manifests are regenerated at the current digest. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
27 lines
12 KiB
YAML
27 lines
12 KiB
YAML
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
|
|
apiVersion: ax.io/v1alpha1
|
|
kind: Workspace
|
|
metadata:
|
|
name: contextforge-prompt-ssti-inputs
|
|
atespace: default
|
|
spec:
|
|
files:
|
|
- path: candidates.json
|
|
content: "{\n \"candidates\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"family\": \"injection\",\n \"id\": \"contextforge-prompt-ssti\"\n}\n"
|
|
- path: functions.json
|
|
content: "[\n {\n \"function\": \"_compile_jinja_template\",\n \"line\": 106,\n \"path\": \"mcpgateway/services/prompt_service.py\"\n },\n {\n \"function\": \"_render_template\",\n \"line\": 2892,\n \"path\": \"mcpgateway/services/prompt_service.py\"\n }\n]\n"
|
|
- path: case.json
|
|
content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.004212,\n \"recorded_usd\": 0.0511\n },\n \"family\": \"injection\",\n \"id\": \"contextforge-prompt-ssti\",\n \"ranges\": {\n \".env.example\": [\n [\n 164,\n 166\n ]\n ],\n \".secrets.baseline\": [\n [\n 6,\n 6\n ],\n [\n 95,\n 95\n ],\n [\n 103,\n 103\n ],\n [\n 111,\n 111\n ],\n [\n 119,\n 119\n ],\n [\n 127,\n 127\n ],\n [\n 135,\n 135\n ],\n [\n 143,\n 143\n ],\n [\n 151,\n 151\n ],\n [\n 159,\n 159\n ],\n [\n 167,\n 167\n ],\n [\n 175,\n 175\n ],\n [\n 183,\n 183\n ],\n [\n 253,\n 253\n ],\n [\n 261,\n 261\n\
|
|
\ ],\n [\n 269,\n 269\n ],\n [\n 277,\n 277\n ],\n [\n 285,\n 285\n ],\n [\n 293,\n 293\n ],\n [\n 301,\n 301\n ],\n [\n 309,\n 309\n ],\n [\n 4215,\n 4215\n ],\n [\n 4865,\n 4865\n ],\n [\n 4873,\n 4873\n ],\n [\n 5195,\n 5195\n ],\n [\n 5957,\n 5957\n ],\n [\n 8393,\n 8393\n ],\n [\n 8401,\n 8401\n ],\n [\n 8409,\n 8409\n ],\n [\n 8759,\n 8759\n ],\n [\n 8767,\n 8767\n ],\n [\n 8775,\n 8775\n ],\n [\n 8783,\n 8783\n ],\n [\n 8791,\n 8791\n ],\n [\n 8799,\n 8799\n ],\n [\n 8807,\n 8807\n ],\n [\n 8815,\n 8815\n ],\n [\n 8823,\n 8823\n ],\n [\n 8831,\n 8831\n ],\n [\n 8839,\n 8839\n ],\n [\n 8847,\n 8847\n ],\n [\n 8855,\n 8855\n ],\n [\n 8863,\n 8863\n ],\n [\n 9237,\n 9237\n ],\n [\n 9245,\n\
|
|
\ 9245\n ],\n [\n 9253,\n 9253\n ],\n [\n 9263,\n 9263\n ]\n ],\n \"CHANGELOG.md\": [\n [\n 4,\n 4\n ]\n ],\n \"README.md\": [\n [\n 751,\n 751\n ],\n [\n 761,\n 761\n ]\n ],\n \"SECURITY.md\": [\n [\n 276,\n 276\n ],\n [\n 374,\n 374\n ]\n ],\n \"crates/mcp_runtime/tests/runtime.rs\": [\n [\n 1647,\n 1647\n ]\n ],\n \"docs/docs/architecture/security-features.md\": [\n [\n 120,\n 120\n ]\n ],\n \"docs/docs/manage/securing.md\": [\n [\n 428,\n 428\n ],\n [\n 440,\n 440\n ]\n ],\n \"mcpgateway/admin.py\": [\n [\n 132,\n 132\n ],\n [\n 13075,\n 13075\n ],\n [\n 13196,\n 13196\n ]\n ],\n \"mcpgateway/config.py\": [\n [\n 93,\n 93\n ],\n [\n 1738,\n 1738\n ],\n [\n 1755,\n 1755\n ]\n ],\n \"mcpgateway/main.py\": [\n [\n 156,\n 156\n ],\n [\n 1820,\n 1823\n ],\n [\n 2426,\n\
|
|
\ 2426\n ],\n [\n 6362,\n 6362\n ],\n [\n 6559,\n 6559\n ]\n ],\n \"mcpgateway/services/content_security.py\": [\n [\n 16,\n 16\n ],\n [\n 48,\n 48\n ],\n [\n 171,\n 171\n ],\n [\n 192,\n 192\n ],\n [\n 194,\n 194\n ],\n [\n 201,\n 201\n ],\n [\n 204,\n 204\n ],\n [\n 333,\n 333\n ],\n [\n 335,\n 335\n ],\n [\n 345,\n 347\n ],\n [\n 351,\n 352\n ],\n [\n 357,\n 357\n ],\n [\n 360,\n 360\n ],\n [\n 362,\n 363\n ]\n ],\n \"mcpgateway/services/metrics.py\": [\n [\n 111,\n 111\n ],\n [\n 121,\n 127\n ],\n [\n 130,\n 131\n ]\n ],\n \"mcpgateway/services/prompt_service.py\": [\n [\n 27,\n 27\n ],\n [\n 53,\n 53\n ],\n [\n 77,\n 77\n ],\n [\n 80,\n 80\n ],\n [\n 82,\n 82\n ],\n [\n 84,\n 84\n ],\n [\n 88,\n 88\n ],\n [\n\
|
|
\ 730,\n 730\n ],\n [\n 767,\n 768\n ],\n [\n 951,\n 951\n ],\n [\n 1014,\n 1014\n ],\n [\n 1101,\n 1102\n ],\n [\n 1228,\n 1228\n ],\n [\n 1275,\n 1275\n ],\n [\n 2169,\n 2169\n ],\n [\n 2276,\n 2276\n ],\n [\n 2278,\n 2278\n ],\n [\n 2446,\n 2446\n ],\n [\n 2889,\n 2889\n ]\n ],\n \"mcpgateway/services/resource_service.py\": [\n [\n 67,\n 67\n ],\n [\n 460,\n 463\n ],\n [\n 529,\n 533\n ],\n [\n 535,\n 535\n ],\n [\n 537,\n 537\n ],\n [\n 685,\n 685\n ],\n [\n 700,\n 700\n ],\n [\n 714,\n 714\n ],\n [\n 838,\n 844\n ],\n [\n 847,\n 847\n ],\n [\n 852,\n 852\n ],\n [\n 872,\n 872\n ],\n [\n 893,\n 893\n ],\n [\n 922,\n 922\n ],\n [\n 2893,\n 2896\n ],\n [\n 2983,\n 2984\n ],\n [\n 2987,\n 3017\n \
|
|
\ ],\n [\n 3018,\n 3018\n ],\n [\n 3029,\n 3029\n ],\n [\n 3037,\n 3037\n ],\n [\n 3204,\n 3204\n ]\n ],\n \"mcpgateway/services/tool_service.py\": [\n [\n 64,\n 74\n ],\n [\n 79,\n 79\n ],\n [\n 949,\n 949\n ],\n [\n 1873,\n 1873\n ],\n [\n 2310,\n 2310\n ],\n [\n 5961,\n 5961\n ]\n ],\n \"tests/e2e/test_admin_apis.py\": [\n [\n 29,\n 29\n ],\n [\n 36,\n 38\n ],\n [\n 116,\n 116\n ],\n [\n 125,\n 127\n ],\n [\n 149,\n 149\n ],\n [\n 151,\n 151\n ],\n [\n 690,\n 690\n ]\n ],\n \"tests/e2e/test_main_apis.py\": [\n [\n 48,\n 48\n ],\n [\n 55,\n 55\n ],\n [\n 58,\n 58\n ],\n [\n 63,\n 63\n ],\n [\n 980,\n 980\n ],\n [\n 1021,\n 1021\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ],\n \"tests/unit/mcpgateway/services/test_content_security.py\"\
|
|
: [\n [\n 5,\n 7\n ],\n [\n 13,\n 13\n ],\n [\n 15,\n 22\n ],\n [\n 49,\n 49\n ],\n [\n 209,\n 209\n ],\n [\n 306,\n 306\n ],\n [\n 323,\n 323\n ],\n [\n 334,\n 336\n ],\n [\n 338,\n 339\n ],\n [\n 342,\n 342\n ],\n [\n 344,\n 345\n ],\n [\n 348,\n 348\n ],\n [\n 363,\n 363\n ],\n [\n 365,\n 365\n ],\n [\n 368,\n 368\n ],\n [\n 373,\n 373\n ],\n [\n 389,\n 389\n ],\n [\n 401,\n 402\n ],\n [\n 404,\n 405\n ],\n [\n 408,\n 408\n ],\n [\n 410,\n 411\n ],\n [\n 414,\n 414\n ],\n [\n 422,\n 422\n ],\n [\n 434,\n 473\n ],\n [\n 479,\n 479\n ],\n [\n 504,\n 504\n ],\n [\n 516,\n 516\n ],\n [\n 526,\n 526\n ],\n [\n 537,\n 537\n ],\n [\n 548,\n 548\n ],\n [\n 582,\n 582\n ],\n [\n \
|
|
\ 614,\n 614\n ]\n ],\n \"tests/unit/mcpgateway/services/test_prompt_service.py\": [\n [\n 33,\n 39\n ],\n [\n 378,\n 378\n ],\n [\n 382,\n 385\n ],\n [\n 387,\n 387\n ],\n [\n 1178,\n 1178\n ],\n [\n 1963,\n 1963\n ],\n [\n 1967,\n 1967\n ],\n [\n 1972,\n 1972\n ],\n [\n 1984,\n 1989\n ],\n [\n 1991,\n 1992\n ]\n ],\n \"tests/unit/mcpgateway/services/test_resource_service.py\": [\n [\n 31,\n 35\n ],\n [\n 2309,\n 2333\n ],\n [\n 2413,\n 2414\n ],\n [\n 2551,\n 2552\n ],\n [\n 2555,\n 2556\n ],\n [\n 2560,\n 2560\n ],\n [\n 2565,\n 2566\n ],\n [\n 2570,\n 2570\n ],\n [\n 2572,\n 2574\n ],\n [\n 2577,\n 2578\n ],\n [\n 2587,\n 2588\n ],\n [\n 2593,\n 2593\n ],\n [\n 2595,\n 2595\n ],\n [\n 2598,\n 2599\n ],\n [\n 2603,\n 2606\n \
|
|
\ ],\n [\n 2608,\n 2608\n ],\n [\n 2610,\n 2612\n ],\n [\n 2614,\n 2615\n ],\n [\n 2618,\n 2619\n ],\n [\n 2622,\n 2622\n ],\n [\n 2637,\n 2638\n ],\n [\n 2642,\n 2642\n ],\n [\n 2644,\n 2645\n ],\n [\n 2724,\n 2753\n ]\n ],\n \"tests/unit/mcpgateway/services/test_row_level_locking.py\": [\n [\n 14,\n 14\n ],\n [\n 16,\n 16\n ],\n [\n 19,\n 24\n ],\n [\n 26,\n 26\n ],\n [\n 77,\n 77\n ],\n [\n 82,\n 82\n ],\n [\n 143,\n 143\n ],\n [\n 442,\n 442\n ],\n [\n 509,\n 509\n ],\n [\n 510,\n 510\n ],\n [\n 515,\n 515\n ],\n [\n 516,\n 516\n ],\n [\n 521,\n 521\n ]\n ],\n \"tests/unit/mcpgateway/test_admin.py\": [\n [\n 281,\n 285\n ],\n [\n 2523,\n 2523\n ],\n [\n 2590,\n 2628\n ],\n [\n 2935,\n 2970\n ]\n ],\n \"tests/unit/mcpgateway/test_main.py\"\
|
|
: [\n [\n 37,\n 49\n ],\n [\n 487,\n 487\n ],\n [\n 490,\n 490\n ],\n [\n 521,\n 521\n ],\n [\n 524,\n 524\n ],\n [\n 764,\n 764\n ],\n [\n 787,\n 787\n ],\n [\n 1198,\n 1198\n ],\n [\n 1208,\n 1208\n ],\n [\n 1211,\n 1211\n ],\n [\n 1414,\n 1414\n ],\n [\n 1416,\n 1416\n ],\n [\n 1420,\n 1420\n ],\n [\n 1432,\n 1432\n ],\n [\n 1441,\n 1441\n ],\n [\n 2407,\n 2407\n ],\n [\n 2476,\n 2476\n ]\n ],\n \"tests/unit/mcpgateway/test_main_error_handlers.py\": [\n [\n 12,\n 13\n ],\n [\n 18,\n 20\n ],\n [\n 21,\n 21\n ],\n [\n 22,\n 22\n ],\n [\n 26,\n 33\n ],\n [\n 44,\n 45\n ],\n [\n 577,\n 577\n ],\n [\n 600,\n 600\n ],\n [\n 607,\n 607\n ],\n [\n 612,\n 612\n ],\n [\n 618,\n 618\n ]\n ]\n },\n \"ranges_basis\"\
|
|
: \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ],\n \"sites_in_set\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ]\n}\n"
|
|
- path: triage.json
|
|
content: "{\n \"candidates\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"kept\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"triage\": {\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\": \"stored\",\n \"mcpgateway/services/prompt_service.py::_render_template\": \"stored\"\n }\n}\n"
|