marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(plane): scoring parity with the reference -- sites, ranges, triage cost

Browse Source 
agree matched declared sites through benchmarks/independent/evaluate.py,
which the runner image does not carry, so site matching would have been
"not assessed". The generator now resolves the fix's old-side ranges
(evaluate.hunks, line for line) into case.json, with the declared sites in
the set and the reference's cost, and agree.matches_declared reproduces
matches_v2; a parity test runs both over the recorded outputs of the 15
validation cases (200 probes, identical verdicts) and agree on those outputs
returns the reference's 16 of 20 and $0.022.

agree reports cost_per_candidate (plane spend) and
cost_per_candidate_with_recorded_triage; Requirement 6.2 is judged on the
latter (design.md). --runner-image re-pins the templates from ops/ax/up.sh's
pin file; the committed manifests are regenerated at the current digest.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-29 19:49:28 +02:00
co-authored by Claude Opus 5.5
parent ac296ee3a4
commit 56b1717246
42 changed files with 591 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.kiro/specs/ai-service-plane/design.md
+11
View File
@@ -133,6 +133,17 @@ artifacts for the increment; the `classify` and `triage` tasks are later specs.
Reads the pass outputs, emits `agreed.json` and `disputed.json` by the same rule as `verify_batched.agreement`,
plus the per-candidate cost, turns and the declared-site match (`evaluate.matches_v2`) for the measurement.
The rule is reproduced in the package (`agree.matches_declared`, parity-tested against `matches_v2` on the recorded
reference outputs) and the generator writes the fix ranges it matches against into each case's `case.json`, so
the runner image carries no benchmark tree.
**Measurement.** Requirement 6.2's cost is judged on `cost_per_candidate_with_recorded_triage`: the plane's two
verify passes plus the recorded triage cost of the case, over the candidates before triage, summed across the run.
The reference's $0.022 (`verifier-batched-check-2026-09-29.json`) paid for per-file triage and the hunts while the
plane applies the recorded triage without a model call, so `cost_per_candidate` (plane spend alone) is reported but
not judged; the triage share is derived per file from the recorded spend minus the priced hunt usage (about $0.087
of the $1.01) and labelled as derived in `case.json`. Sites are judged as `declared_sites_matched_agreed` against
`declared_sites_in_set`, the reference's 16 of 20.
## Data Models
plane/runs/validation-46.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
# 15 cases, 46 candidates (43 kept by the recorded triage); usd ceiling 2.02.
apiVersion: openultrasast.io/v1alpha1
kind: Run
plane/tasks/agree.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ metadata:
name: agree
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3"
image: "localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4"
command: ["agree"]
resources:
requests:
plane/tasks/repo-facts.yaml
+1 -1
View File
@@ -8,7 +8,7 @@ metadata:
name: repo-facts
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3"
image: "localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4"
command: ["repo-facts"]
resources:
requests:
plane/tasks/validation-46.yaml
+61 -61
View File
@@ -1,11 +1,11 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Task
metadata:
name: repo-facts-budibase-mongo-template-nosqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -34,7 +34,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -65,7 +65,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -94,7 +94,7 @@ metadata:
name: agree-budibase-mongo-template-nosqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -121,7 +121,7 @@ metadata:
name: repo-facts-cct-studio-server-cmdi
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -150,7 +150,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -181,7 +181,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -210,7 +210,7 @@ metadata:
name: agree-cct-studio-server-cmdi
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -237,7 +237,7 @@ metadata:
name: repo-facts-contextforge-prompt-ssti
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -266,7 +266,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -297,7 +297,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -326,7 +326,7 @@ metadata:
name: agree-contextforge-prompt-ssti
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -353,7 +353,7 @@ metadata:
name: repo-facts-froxlor-admin-ip-second-order-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -382,7 +382,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -413,7 +413,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -442,7 +442,7 @@ metadata:
name: agree-froxlor-admin-ip-second-order-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -469,7 +469,7 @@ metadata:
name: repo-facts-fuxa-tdengine-escape-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -498,7 +498,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -529,7 +529,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -558,7 +558,7 @@ metadata:
name: agree-fuxa-tdengine-escape-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -585,7 +585,7 @@ metadata:
name: repo-facts-lmdeploy-media-url-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -614,7 +614,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -645,7 +645,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -674,7 +674,7 @@ metadata:
name: agree-lmdeploy-media-url-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -701,7 +701,7 @@ metadata:
name: repo-facts-medplum-external-auth-open-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -730,7 +730,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -761,7 +761,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -790,7 +790,7 @@ metadata:
name: agree-medplum-external-auth-open-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -817,7 +817,7 @@ metadata:
name: repo-facts-pgadmin-maintenance-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -846,7 +846,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -877,7 +877,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -906,7 +906,7 @@ metadata:
name: agree-pgadmin-maintenance-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -933,7 +933,7 @@ metadata:
name: repo-facts-qwed-sympy-eval
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -962,7 +962,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -993,7 +993,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1022,7 +1022,7 @@ metadata:
name: agree-qwed-sympy-eval
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -1049,7 +1049,7 @@ metadata:
name: repo-facts-reactpress-findall-column-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -1078,7 +1078,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1109,7 +1109,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1138,7 +1138,7 @@ metadata:
name: agree-reactpress-findall-column-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -1165,7 +1165,7 @@ metadata:
name: repo-facts-unleash-addon-webhook-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -1194,7 +1194,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1225,7 +1225,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1254,7 +1254,7 @@ metadata:
name: agree-unleash-addon-webhook-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -1281,7 +1281,7 @@ metadata:
name: repo-facts-winml-cli-serve-cors
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -1310,7 +1310,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1341,7 +1341,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1370,7 +1370,7 @@ metadata:
name: agree-winml-cli-serve-cors
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -1397,7 +1397,7 @@ metadata:
name: repo-facts-wp-directorist-avatar-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -1426,7 +1426,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1457,7 +1457,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1486,7 +1486,7 @@ metadata:
name: agree-wp-directorist-avatar-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -1513,7 +1513,7 @@ metadata:
name: repo-facts-wp-gopay-log-filter-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -1542,7 +1542,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1573,7 +1573,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1602,7 +1602,7 @@ metadata:
name: agree-wp-gopay-log-filter-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
@@ -1629,7 +1629,7 @@ metadata:
name: repo-facts-wp-groundhogg-confirm-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- repo-facts
env:
@@ -1658,7 +1658,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1689,7 +1689,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- verify
env:
@@ -1718,7 +1718,7 @@ metadata:
name: agree-wp-groundhogg-confirm-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3
image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4
command:
- agree
env:
plane/tasks/verify.yaml
+1 -1
View File
@@ -10,7 +10,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: "localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3"
image: "localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4"
command: ["verify"]
resources:
requests:
plane/workspaces/budibase-mongo-template-nosqli-inputs.yaml
+3 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,7 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"read\",\n \"line\": 539,\n \"path\": \"packages/server/src/integrations/mongodb.ts\"\n },\n {\n \"function\": \"enrichContext\",\n \"line\": 136,\n \"path\": \"packages/server/src/sdk/workspace/queries/queries.ts\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"budibase-mongo-template-nosqli\",\n \"sites\": [\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\",\n \"packages/server/src/integrations/mongodb.ts::read\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.003638,\n \"recorded_usd\": 0.0767\n },\n \"family\": \"injection\",\n \"id\": \"budibase-mongo-template-nosqli\",\n \"ranges\": {\n \"packages/server/src/api/routes/tests/queries/rest.spec.ts\": [\n [\n 580,\n 580\n ]\n ],\n \"packages/server/src/sdk/workspace/queries/queries.ts\": [\n [\n 2,\n 3\n ],\n [\n 15,\n 15\n ],\n [\n 120,\n 125\n ],\n [\n 136,\n 136\n ],\n [\n 138,\n 140\n ]\n ],\n \"packages/string-templates/src/index.ts\": [\n [\n 276,\n 276\n ]\n ],\n \"packages/string-templates/test/basic.spec.ts\": [\n [\n 10,\n 10\n ],\n [\n 128,\n 128\n ]\n ]\n },\n \"ranges_basis\"\
: \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\",\n \"packages/server/src/integrations/mongodb.ts::read\"\n ],\n \"sites_in_set\": [\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\",\n \"packages/server/src/integrations/mongodb.ts::read\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"packages/server/src/integrations/mongodb.ts\",\n \"read\",\n 539\n ],\n [\n \"packages/server/src/sdk/workspace/queries/queries.ts\",\n \"enrichContext\",\n 136\n ]\n ],\n \"kept\": [\n [\n \"packages/server/src/integrations/mongodb.ts\",\n \"read\",\n 539\n ],\n [\n \"packages/server/src/sdk/workspace/queries/queries.ts\",\n \"enrichContext\",\n 136\n ]\n ],\n \"triage\": {\n \"packages/server/src/integrations/mongodb.ts::read\": \"request\",\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\": \"parameter\"\n }\n}\n"
plane/workspaces/budibase-mongo-template-nosqli-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/cct-studio-server-cmdi-inputs.yaml
+3 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,7 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"checkAndInstallAgent\",\n \"line\": 280,\n \"path\": \"cli-tool/src/sandbox-server.js\"\n },\n {\n \"function\": \"executeLocalTask\",\n \"line\": 464,\n \"path\": \"cli-tool/src/sandbox-server.js\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"cct-studio-server-cmdi\",\n \"sites\": [\n \"cli-tool/src/sandbox-server.js::executeLocalTask\",\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.003055,\n \"recorded_usd\": 0.0242\n },\n \"family\": \"injection\",\n \"id\": \"cct-studio-server-cmdi\",\n \"ranges\": {\n \"CHANGELOG.md\": [\n [\n 7,\n 7\n ]\n ],\n \"cli-tool/package.json\": [\n [\n 3,\n 3\n ]\n ],\n \"cli-tool/src/sandbox-server.js\": [\n [\n 11,\n 11\n ],\n [\n 50,\n 50\n ],\n [\n 52,\n 52\n ],\n [\n 54,\n 54\n ],\n [\n 202,\n 202\n ],\n [\n 209,\n 209\n ],\n [\n 212,\n 214\n ],\n [\n 216,\n 217\n ],\n [\n 278,\n 278\n ],\n [\n 280,\n 280\n ],\n [\n 282,\n 283\n ],\n [\n 463,\n 464\n ],\n [\n 471,\n \
\ 471\n ],\n [\n 561,\n 562\n ]\n ],\n \"package.json\": [\n [\n 3,\n 3\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"cli-tool/src/sandbox-server.js::executeLocalTask\",\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\"\n ],\n \"sites_in_set\": [\n \"cli-tool/src/sandbox-server.js::executeLocalTask\",\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"cli-tool/src/sandbox-server.js\",\n \"checkAndInstallAgent\",\n 280\n ],\n [\n \"cli-tool/src/sandbox-server.js\",\n \"executeLocalTask\",\n 464\n ]\n ],\n \"kept\": [\n [\n \"cli-tool/src/sandbox-server.js\",\n \"checkAndInstallAgent\",\n 280\n ],\n [\n \"cli-tool/src/sandbox-server.js\",\n \"executeLocalTask\",\n 464\n ]\n ],\n \"triage\": {\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\": \"parameter\",\n \"cli-tool/src/sandbox-server.js::executeLocalTask\": \"request\"\n }\n}\n"
plane/workspaces/cct-studio-server-cmdi-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/contextforge-prompt-ssti-inputs.yaml
+12 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,16 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"_compile_jinja_template\",\n \"line\": 106,\n \"path\": \"mcpgateway/services/prompt_service.py\"\n },\n {\n \"function\": \"_render_template\",\n \"line\": 2892,\n \"path\": \"mcpgateway/services/prompt_service.py\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"contextforge-prompt-ssti\",\n \"sites\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.004212,\n \"recorded_usd\": 0.0511\n },\n \"family\": \"injection\",\n \"id\": \"contextforge-prompt-ssti\",\n \"ranges\": {\n \".env.example\": [\n [\n 164,\n 166\n ]\n ],\n \".secrets.baseline\": [\n [\n 6,\n 6\n ],\n [\n 95,\n 95\n ],\n [\n 103,\n 103\n ],\n [\n 111,\n 111\n ],\n [\n 119,\n 119\n ],\n [\n 127,\n 127\n ],\n [\n 135,\n 135\n ],\n [\n 143,\n 143\n ],\n [\n 151,\n 151\n ],\n [\n 159,\n 159\n ],\n [\n 167,\n 167\n ],\n [\n 175,\n 175\n ],\n [\n 183,\n 183\n ],\n [\n 253,\n 253\n ],\n [\n 261,\n 261\n\
\ ],\n [\n 269,\n 269\n ],\n [\n 277,\n 277\n ],\n [\n 285,\n 285\n ],\n [\n 293,\n 293\n ],\n [\n 301,\n 301\n ],\n [\n 309,\n 309\n ],\n [\n 4215,\n 4215\n ],\n [\n 4865,\n 4865\n ],\n [\n 4873,\n 4873\n ],\n [\n 5195,\n 5195\n ],\n [\n 5957,\n 5957\n ],\n [\n 8393,\n 8393\n ],\n [\n 8401,\n 8401\n ],\n [\n 8409,\n 8409\n ],\n [\n 8759,\n 8759\n ],\n [\n 8767,\n 8767\n ],\n [\n 8775,\n 8775\n ],\n [\n 8783,\n 8783\n ],\n [\n 8791,\n 8791\n ],\n [\n 8799,\n 8799\n ],\n [\n 8807,\n 8807\n ],\n [\n 8815,\n 8815\n ],\n [\n 8823,\n 8823\n ],\n [\n 8831,\n 8831\n ],\n [\n 8839,\n 8839\n ],\n [\n 8847,\n 8847\n ],\n [\n 8855,\n 8855\n ],\n [\n 8863,\n 8863\n ],\n [\n 9237,\n 9237\n ],\n [\n 9245,\n\
\ 9245\n ],\n [\n 9253,\n 9253\n ],\n [\n 9263,\n 9263\n ]\n ],\n \"CHANGELOG.md\": [\n [\n 4,\n 4\n ]\n ],\n \"README.md\": [\n [\n 751,\n 751\n ],\n [\n 761,\n 761\n ]\n ],\n \"SECURITY.md\": [\n [\n 276,\n 276\n ],\n [\n 374,\n 374\n ]\n ],\n \"crates/mcp_runtime/tests/runtime.rs\": [\n [\n 1647,\n 1647\n ]\n ],\n \"docs/docs/architecture/security-features.md\": [\n [\n 120,\n 120\n ]\n ],\n \"docs/docs/manage/securing.md\": [\n [\n 428,\n 428\n ],\n [\n 440,\n 440\n ]\n ],\n \"mcpgateway/admin.py\": [\n [\n 132,\n 132\n ],\n [\n 13075,\n 13075\n ],\n [\n 13196,\n 13196\n ]\n ],\n \"mcpgateway/config.py\": [\n [\n 93,\n 93\n ],\n [\n 1738,\n 1738\n ],\n [\n 1755,\n 1755\n ]\n ],\n \"mcpgateway/main.py\": [\n [\n 156,\n 156\n ],\n [\n 1820,\n 1823\n ],\n [\n 2426,\n\
\ 2426\n ],\n [\n 6362,\n 6362\n ],\n [\n 6559,\n 6559\n ]\n ],\n \"mcpgateway/services/content_security.py\": [\n [\n 16,\n 16\n ],\n [\n 48,\n 48\n ],\n [\n 171,\n 171\n ],\n [\n 192,\n 192\n ],\n [\n 194,\n 194\n ],\n [\n 201,\n 201\n ],\n [\n 204,\n 204\n ],\n [\n 333,\n 333\n ],\n [\n 335,\n 335\n ],\n [\n 345,\n 347\n ],\n [\n 351,\n 352\n ],\n [\n 357,\n 357\n ],\n [\n 360,\n 360\n ],\n [\n 362,\n 363\n ]\n ],\n \"mcpgateway/services/metrics.py\": [\n [\n 111,\n 111\n ],\n [\n 121,\n 127\n ],\n [\n 130,\n 131\n ]\n ],\n \"mcpgateway/services/prompt_service.py\": [\n [\n 27,\n 27\n ],\n [\n 53,\n 53\n ],\n [\n 77,\n 77\n ],\n [\n 80,\n 80\n ],\n [\n 82,\n 82\n ],\n [\n 84,\n 84\n ],\n [\n 88,\n 88\n ],\n [\n\
\ 730,\n 730\n ],\n [\n 767,\n 768\n ],\n [\n 951,\n 951\n ],\n [\n 1014,\n 1014\n ],\n [\n 1101,\n 1102\n ],\n [\n 1228,\n 1228\n ],\n [\n 1275,\n 1275\n ],\n [\n 2169,\n 2169\n ],\n [\n 2276,\n 2276\n ],\n [\n 2278,\n 2278\n ],\n [\n 2446,\n 2446\n ],\n [\n 2889,\n 2889\n ]\n ],\n \"mcpgateway/services/resource_service.py\": [\n [\n 67,\n 67\n ],\n [\n 460,\n 463\n ],\n [\n 529,\n 533\n ],\n [\n 535,\n 535\n ],\n [\n 537,\n 537\n ],\n [\n 685,\n 685\n ],\n [\n 700,\n 700\n ],\n [\n 714,\n 714\n ],\n [\n 838,\n 844\n ],\n [\n 847,\n 847\n ],\n [\n 852,\n 852\n ],\n [\n 872,\n 872\n ],\n [\n 893,\n 893\n ],\n [\n 922,\n 922\n ],\n [\n 2893,\n 2896\n ],\n [\n 2983,\n 2984\n ],\n [\n 2987,\n 3017\n \
\ ],\n [\n 3018,\n 3018\n ],\n [\n 3029,\n 3029\n ],\n [\n 3037,\n 3037\n ],\n [\n 3204,\n 3204\n ]\n ],\n \"mcpgateway/services/tool_service.py\": [\n [\n 64,\n 74\n ],\n [\n 79,\n 79\n ],\n [\n 949,\n 949\n ],\n [\n 1873,\n 1873\n ],\n [\n 2310,\n 2310\n ],\n [\n 5961,\n 5961\n ]\n ],\n \"tests/e2e/test_admin_apis.py\": [\n [\n 29,\n 29\n ],\n [\n 36,\n 38\n ],\n [\n 116,\n 116\n ],\n [\n 125,\n 127\n ],\n [\n 149,\n 149\n ],\n [\n 151,\n 151\n ],\n [\n 690,\n 690\n ]\n ],\n \"tests/e2e/test_main_apis.py\": [\n [\n 48,\n 48\n ],\n [\n 55,\n 55\n ],\n [\n 58,\n 58\n ],\n [\n 63,\n 63\n ],\n [\n 980,\n 980\n ],\n [\n 1021,\n 1021\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ],\n \"tests/unit/mcpgateway/services/test_content_security.py\"\
: [\n [\n 5,\n 7\n ],\n [\n 13,\n 13\n ],\n [\n 15,\n 22\n ],\n [\n 49,\n 49\n ],\n [\n 209,\n 209\n ],\n [\n 306,\n 306\n ],\n [\n 323,\n 323\n ],\n [\n 334,\n 336\n ],\n [\n 338,\n 339\n ],\n [\n 342,\n 342\n ],\n [\n 344,\n 345\n ],\n [\n 348,\n 348\n ],\n [\n 363,\n 363\n ],\n [\n 365,\n 365\n ],\n [\n 368,\n 368\n ],\n [\n 373,\n 373\n ],\n [\n 389,\n 389\n ],\n [\n 401,\n 402\n ],\n [\n 404,\n 405\n ],\n [\n 408,\n 408\n ],\n [\n 410,\n 411\n ],\n [\n 414,\n 414\n ],\n [\n 422,\n 422\n ],\n [\n 434,\n 473\n ],\n [\n 479,\n 479\n ],\n [\n 504,\n 504\n ],\n [\n 516,\n 516\n ],\n [\n 526,\n 526\n ],\n [\n 537,\n 537\n ],\n [\n 548,\n 548\n ],\n [\n 582,\n 582\n ],\n [\n \
\ 614,\n 614\n ]\n ],\n \"tests/unit/mcpgateway/services/test_prompt_service.py\": [\n [\n 33,\n 39\n ],\n [\n 378,\n 378\n ],\n [\n 382,\n 385\n ],\n [\n 387,\n 387\n ],\n [\n 1178,\n 1178\n ],\n [\n 1963,\n 1963\n ],\n [\n 1967,\n 1967\n ],\n [\n 1972,\n 1972\n ],\n [\n 1984,\n 1989\n ],\n [\n 1991,\n 1992\n ]\n ],\n \"tests/unit/mcpgateway/services/test_resource_service.py\": [\n [\n 31,\n 35\n ],\n [\n 2309,\n 2333\n ],\n [\n 2413,\n 2414\n ],\n [\n 2551,\n 2552\n ],\n [\n 2555,\n 2556\n ],\n [\n 2560,\n 2560\n ],\n [\n 2565,\n 2566\n ],\n [\n 2570,\n 2570\n ],\n [\n 2572,\n 2574\n ],\n [\n 2577,\n 2578\n ],\n [\n 2587,\n 2588\n ],\n [\n 2593,\n 2593\n ],\n [\n 2595,\n 2595\n ],\n [\n 2598,\n 2599\n ],\n [\n 2603,\n 2606\n \
\ ],\n [\n 2608,\n 2608\n ],\n [\n 2610,\n 2612\n ],\n [\n 2614,\n 2615\n ],\n [\n 2618,\n 2619\n ],\n [\n 2622,\n 2622\n ],\n [\n 2637,\n 2638\n ],\n [\n 2642,\n 2642\n ],\n [\n 2644,\n 2645\n ],\n [\n 2724,\n 2753\n ]\n ],\n \"tests/unit/mcpgateway/services/test_row_level_locking.py\": [\n [\n 14,\n 14\n ],\n [\n 16,\n 16\n ],\n [\n 19,\n 24\n ],\n [\n 26,\n 26\n ],\n [\n 77,\n 77\n ],\n [\n 82,\n 82\n ],\n [\n 143,\n 143\n ],\n [\n 442,\n 442\n ],\n [\n 509,\n 509\n ],\n [\n 510,\n 510\n ],\n [\n 515,\n 515\n ],\n [\n 516,\n 516\n ],\n [\n 521,\n 521\n ]\n ],\n \"tests/unit/mcpgateway/test_admin.py\": [\n [\n 281,\n 285\n ],\n [\n 2523,\n 2523\n ],\n [\n 2590,\n 2628\n ],\n [\n 2935,\n 2970\n ]\n ],\n \"tests/unit/mcpgateway/test_main.py\"\
: [\n [\n 37,\n 49\n ],\n [\n 487,\n 487\n ],\n [\n 490,\n 490\n ],\n [\n 521,\n 521\n ],\n [\n 524,\n 524\n ],\n [\n 764,\n 764\n ],\n [\n 787,\n 787\n ],\n [\n 1198,\n 1198\n ],\n [\n 1208,\n 1208\n ],\n [\n 1211,\n 1211\n ],\n [\n 1414,\n 1414\n ],\n [\n 1416,\n 1416\n ],\n [\n 1420,\n 1420\n ],\n [\n 1432,\n 1432\n ],\n [\n 1441,\n 1441\n ],\n [\n 2407,\n 2407\n ],\n [\n 2476,\n 2476\n ]\n ],\n \"tests/unit/mcpgateway/test_main_error_handlers.py\": [\n [\n 12,\n 13\n ],\n [\n 18,\n 20\n ],\n [\n 21,\n 21\n ],\n [\n 22,\n 22\n ],\n [\n 26,\n 33\n ],\n [\n 44,\n 45\n ],\n [\n 577,\n 577\n ],\n [\n 600,\n 600\n ],\n [\n 607,\n 607\n ],\n [\n 612,\n 612\n ],\n [\n 618,\n 618\n ]\n ]\n },\n \"ranges_basis\"\
: \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ],\n \"sites_in_set\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"kept\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"triage\": {\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\": \"stored\",\n \"mcpgateway/services/prompt_service.py::_render_template\": \"stored\"\n }\n}\n"
plane/workspaces/contextforge-prompt-ssti-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/froxlor-admin-ip-second-order-sqli-inputs.yaml
+3 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,7 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"validateIpAddresses\",\n \"line\": 1050,\n \"path\": \"lib/Froxlor/Api/Commands/Domains.php\"\n },\n {\n \"function\": \"listing\",\n \"line\": 76,\n \"path\": \"lib/Froxlor/Api/Commands/IpsAndPorts.php\"\n },\n {\n \"function\": \"listingCount\",\n \"line\": 108,\n \"path\": \"lib/Froxlor/Api/Commands/IpsAndPorts.php\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"froxlor-admin-ip-second-order-sqli\",\n \"sites\": [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\",\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 3,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.007758,\n \"recorded_usd\": 0.0873\n },\n \"family\": \"injection\",\n \"id\": \"froxlor-admin-ip-second-order-sqli\",\n \"ranges\": {\n \"lib/Froxlor/Api/Commands/Admins.php\": [\n [\n 273,\n 273\n ],\n [\n 358,\n 358\n ],\n [\n 619,\n 619\n ],\n [\n 729,\n 729\n ]\n ],\n \"lib/Froxlor/Api/Commands/Domains.php\": [\n [\n 1016,\n 1016\n ]\n ],\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\": [\n [\n 72,\n 72\n ],\n [\n 104,\n 104\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\"\
,\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\",\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\"\n ],\n \"sites_in_set\": [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\",\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"lib/Froxlor/Api/Commands/Domains.php\",\n \"validateIpAddresses\",\n 1050\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listing\",\n 76\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listingCount\",\n 108\n ]\n ],\n \"kept\": [\n [\n \"lib/Froxlor/Api/Commands/Domains.php\",\n \"validateIpAddresses\",\n 1050\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listing\",\n 76\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listingCount\",\n 108\n ]\n ],\n \"triage\": {\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\": \"parameter\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\": \"request\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\": \"request\"\n }\n}\n"
plane/workspaces/froxlor-admin-ip-second-order-sqli-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/fuxa-tdengine-escape-sqli-inputs.yaml
+2 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,6 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"getDaqValue\",\n \"line\": 89,\n \"path\": \"server/runtime/storage/tdengine/index.js\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"fuxa-tdengine-escape-sqli\",\n \"sites\": [\n \"server/runtime/storage/tdengine/index.js::getDaqValue\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.000833,\n \"recorded_usd\": 0.1527\n },\n \"family\": \"injection\",\n \"id\": \"fuxa-tdengine-escape-sqli\",\n \"ranges\": {\n \"server/runtime/storage/tdengine/index.js\": [\n [\n 11,\n 11\n ],\n [\n 121,\n 121\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"server/runtime/storage/tdengine/index.js::getDaqValue\"\n ],\n \"sites_in_set\": [\n \"server/runtime/storage/tdengine/index.js::getDaqValue\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"server/runtime/storage/tdengine/index.js\",\n \"getDaqValue\",\n 89\n ]\n ],\n \"kept\": [\n [\n \"server/runtime/storage/tdengine/index.js\",\n \"getDaqValue\",\n 89\n ]\n ],\n \"triage\": {\n \"server/runtime/storage/tdengine/index.js::getDaqValue\": \"parameter\"\n }\n}\n"
plane/workspaces/fuxa-tdengine-escape-sqli-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/lmdeploy-media-url-ssrf-inputs.yaml
+5 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,9 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"_load_http_url\",\n \"line\": 68,\n \"path\": \"lmdeploy/vl/media/connection.py\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"lmdeploy-media-url-ssrf\",\n \"sites\": [\n \"lmdeploy/vl/media/connection.py::_load_http_url\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.000768,\n \"recorded_usd\": 0.0249\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"lmdeploy-media-url-ssrf\",\n \"ranges\": {\n \"docs/en/multi_modal/multimodal_inputs.md\": [\n [\n 395,\n 395\n ]\n ],\n \"docs/zh_cn/multi_modal/multimodal_inputs.md\": [\n [\n 394,\n 394\n ]\n ],\n \"lmdeploy/api.py\": [\n [\n 21,\n 21\n ],\n [\n 45,\n 45\n ],\n [\n 78,\n 78\n ]\n ],\n \"lmdeploy/cli/serve.py\": [\n [\n 77,\n 77\n ],\n [\n 325,\n 325\n ],\n [\n 358,\n 358\n ]\n ],\n \"lmdeploy/pipeline.py\": [\n [\n 42,\n 42\n ],\n [\n 53,\n 53\n ],\n [\n 84,\n 84\n\
\ ],\n [\n 89,\n 89\n ],\n [\n 115,\n 115\n ],\n [\n 165,\n 165\n ],\n [\n 204,\n 204\n ]\n ],\n \"lmdeploy/serve/core/async_engine.py\": [\n [\n 116,\n 116\n ],\n [\n 125,\n 125\n ]\n ],\n \"lmdeploy/serve/core/vl_async_engine.py\": [\n [\n 20,\n 20\n ],\n [\n 46,\n 46\n ],\n [\n 52,\n 52\n ]\n ],\n \"lmdeploy/serve/openai/api_server.py\": [\n [\n 1528,\n 1528\n ],\n [\n 1580,\n 1580\n ],\n [\n 1620,\n 1620\n ]\n ],\n \"lmdeploy/serve/processors/multimodal.py\": [\n [\n 40,\n 40\n ],\n [\n 47,\n 47\n ],\n [\n 52,\n 52\n ],\n [\n 106,\n 107\n ],\n [\n 156,\n 156\n ],\n [\n 163,\n 163\n ],\n [\n 167,\n 167\n ],\n [\n 170,\n 170\n ],\n [\n 180,\n 180\n ],\n [\n 192,\n 192\n ],\n [\n 269,\n 269\n ],\n [\n 282,\n 282\n ],\n [\n 313,\n\
\ 313\n ],\n [\n 315,\n 316\n ],\n [\n 329,\n 329\n ],\n [\n 403,\n 403\n ]\n ],\n \"lmdeploy/vl/media/connection.py\": [\n [\n 7,\n 7\n ],\n [\n 10,\n 10\n ],\n [\n 28,\n 28\n ],\n [\n 34,\n 34\n ],\n [\n 54,\n 54\n ],\n [\n 56,\n 59\n ],\n [\n 67,\n 69\n ],\n [\n 71,\n 71\n ],\n [\n 95,\n 95\n ],\n [\n 100,\n 100\n ]\n ],\n \"tests/test_lmdeploy/test_content_merge.py\": [\n [\n 12,\n 12\n ],\n [\n 245,\n 245\n ],\n [\n 307,\n 307\n ],\n [\n 335,\n 335\n ],\n [\n 340,\n 340\n ],\n [\n 385,\n 385\n ],\n [\n 407,\n 407\n ],\n [\n 415,\n 415\n ]\n ],\n \"tests/test_lmdeploy/test_vl/test_safe_url.py\": [\n [\n 43,\n 44\n ],\n [\n 47,\n 47\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case,\
\ 'old')\",\n \"sites\": [\n \"lmdeploy/vl/media/connection.py::_load_http_url\"\n ],\n \"sites_in_set\": [\n \"lmdeploy/vl/media/connection.py::_load_http_url\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"lmdeploy/vl/media/connection.py\",\n \"_load_http_url\",\n 68\n ]\n ],\n \"kept\": [\n [\n \"lmdeploy/vl/media/connection.py\",\n \"_load_http_url\",\n 68\n ]\n ],\n \"triage\": {\n \"lmdeploy/vl/media/connection.py::_load_http_url\": \"parameter\"\n }\n}\n"
plane/workspaces/lmdeploy-media-url-ssrf-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/medplum-external-auth-open-redirect-inputs.yaml
+4 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,8 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"externalCallbackHandler\",\n \"line\": 166,\n \"path\": \"packages/server/src/auth/external.ts\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"medplum-external-auth-open-redirect\",\n \"sites\": [\n \"packages/server/src/auth/external.ts::externalCallbackHandler\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.001509,\n \"recorded_usd\": 0.0228\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"medplum-external-auth-open-redirect\",\n \"ranges\": {\n \"packages/definitions/src/fhir/r4/profiles-medplum.json\": [\n [\n 890,\n 904\n ],\n [\n 1080,\n 1080\n ]\n ],\n \"packages/docs/static/data/medplumDefinitions/clientapplication.json\": [\n [\n 308,\n 308\n ],\n [\n 315,\n 315\n ],\n [\n 317,\n 317\n ],\n [\n 319,\n 319\n ],\n [\n 324,\n 324\n ],\n [\n 331,\n 331\n ],\n [\n 333,\n 333\n ],\n [\n 335,\n 335\n ],\n [\n 340,\n 340\n ],\n [\n 344,\n 344\n ],\n\
\ [\n 347,\n 347\n ],\n [\n 348,\n 348\n ],\n [\n 351,\n 351\n ],\n [\n 450,\n 450\n ]\n ],\n \"packages/docs/static/data/medplumDefinitions/smartapplaunch.json\": [\n [\n 206,\n 206\n ]\n ],\n \"packages/fhirtypes/dist/ClientApplication.d.ts\": [\n [\n 137,\n 141\n ],\n [\n 154,\n 154\n ],\n [\n 191,\n 191\n ],\n [\n 195,\n 195\n ],\n [\n 197,\n 197\n ],\n [\n 200,\n 200\n ],\n [\n 202,\n 202\n ],\n [\n 205,\n 205\n ],\n [\n 207,\n 207\n ],\n [\n 211,\n 211\n ],\n [\n 213,\n 213\n ],\n [\n 216,\n 216\n ],\n [\n 218,\n 218\n ],\n [\n 221,\n 221\n ],\n [\n 223,\n 223\n ]\n ],\n \"packages/server/src/auth/external.test.ts\": [\n [\n 410,\n 410\n ]\n ],\n \"packages/server/src/auth/external.ts\": [\n [\n 13,\n 13\n ],\n [\n 18,\n 18\n ],\n [\n 124,\n\
\ 133\n ],\n [\n 139,\n 139\n ],\n [\n 283,\n 283\n ]\n ],\n \"packages/server/src/oauth/clients.ts\": [\n [\n 69,\n 69\n ],\n [\n 76,\n 76\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"packages/server/src/auth/external.ts::externalCallbackHandler\"\n ],\n \"sites_in_set\": [\n \"packages/server/src/auth/external.ts::externalCallbackHandler\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"packages/server/src/auth/external.ts\",\n \"externalCallbackHandler\",\n 166\n ]\n ],\n \"kept\": [\n [\n \"packages/server/src/auth/external.ts\",\n \"externalCallbackHandler\",\n 166\n ]\n ],\n \"triage\": {\n \"packages/server/src/auth/external.ts::externalCallbackHandler\": \"parameter\"\n }\n}\n"
plane/workspaces/medplum-external-auth-open-redirect-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/pgadmin-maintenance-sqli-inputs.yaml
+3 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -17,7 +17,8 @@ spec:
\n },\n {\n \"function\": \"run_job\",\n \"line\": 746,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\"\n },\n {\n \"function\": \"create\",\n \"line\": 365,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_schedules/__init__.py\"\n },\n {\n \"function\": \"msql\",\n \"line\": 454,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_schedules/__init__.py\"\n },\n {\n \"function\": \"sql\",\n \"line\": 492,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_schedules/__init__.py\"\n },\n {\n \"function\": \"msql\",\n \"line\": 623,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/event_triggers/__init__.py\"\n },\n {\n \"function\": \"create\",\n \"line\": 290,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/extensions/__init__.py\"\n },\n {\n \"function\"\
: \"delete\",\n \"line\": 394,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/extensions/__init__.py\"\n },\n {\n \"function\": \"get_extension_details\",\n \"line\": 30,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/extensions/utils.py\"\n },\n {\n \"function\": \"delete\",\n \"line\": 601,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/foreign_data_wrappers/__init__.py\"\n },\n {\n \"function\": \"create\",\n \"line\": 464,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/foreign_data_wrappers/foreign_servers/__init__.py\"\n },\n {\n \"function\": \"create_maintenance_job\",\n \"line\": 222,\n \"path\": \"web/pgadmin/tools/maintenance/__init__.py\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"pgadmin-maintenance-sqli\",\n \"sites\": [\n \"web/pgadmin/tools/maintenance/__init__.py::create_maintenance_job\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 27,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.048665,\n \"recorded_usd\": 0.3949\n },\n \"family\": \"injection\",\n \"id\": \"pgadmin-maintenance-sqli\",\n \"ranges\": {\n \"web/pgadmin/tools/maintenance/__init__.py\": [\n [\n 12,\n 12\n ],\n [\n 150,\n 150\n ],\n [\n 171,\n 171\n ]\n ],\n \"web/pgadmin/tools/maintenance/templates/maintenance/sql/command.sql\": [\n [\n 16,\n 16\n ]\n ],\n \"web/pgadmin/tools/maintenance/tests/test_maintenance_create_job_unit_test.py\": [\n [\n 562,\n 562\n ],\n [\n 647,\n 647\n ],\n [\n 714,\n 714\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case,\
\ 'old')\",\n \"sites\": [\n \"web/pgadmin/tools/maintenance/__init__.py::create_maintenance_job\"\n ],\n \"sites_in_set\": [\n \"web/pgadmin/tools/maintenance/__init__.py::create_maintenance_job\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"web/pgadmin/authenticate/ldap.py\",\n \"login\",\n 169\n ],\n [\n \"web/pgadmin/authenticate/ldap.py\",\n \"search_ldap_user\",\n 288\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/__init__.py\",\n \"create_restore_point\",\n 1881\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"_check_rename_db_or_change_table_space\",\n 808\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"_fetch_db_details\",\n 831\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"create\",\n 761\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"delete\",\n 1061\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"get_ctypes\",\n 651\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"get_encodings\",\n\
\ 623\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"get_sql\",\n 1107\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"update\",\n 936\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/casts/__init__.py\",\n \"delete\",\n 546\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"_fetch_properties\",\n 362\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"create\",\n 412\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"enable_disable\",\n 662\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"get_job_args_value\",\n 779\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\"\
plane/workspaces/pgadmin-maintenance-sqli-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/qwed-sympy-eval-inputs.yaml
+3 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,7 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"_verify_item\",\n \"line\": 239,\n \"path\": \"src/qwed_new/core/batch.py\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"qwed-sympy-eval\",\n \"sites\": [\n \"src/qwed_new/api/main.py::verify_math\",\n \"src/qwed_new/core/batch.py::_verify_item\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.00149,\n \"recorded_usd\": 0.022\n },\n \"family\": \"injection\",\n \"id\": \"qwed-sympy-eval\",\n \"ranges\": {\n \"src/qwed_new/api/main.py\": [\n [\n 460,\n 460\n ],\n [\n 475,\n 476\n ],\n [\n 504,\n 504\n ]\n ],\n \"src/qwed_new/core/batch.py\": [\n [\n 223,\n 223\n ],\n [\n 229,\n 230\n ],\n [\n 239,\n 239\n ],\n [\n 0,\n 0\n ]\n ],\n \"src/qwed_new/core/validator.py\": [\n [\n 13,\n 13\n ],\n [\n 64,\n 64\n ]\n ],\n \"src/qwed_new/core/verifier.py\": [\n [\n 21,\n 21\n ],\n [\n 123,\n 123\n ],\n [\n 200,\n 201\n ],\n [\n 284,\n 286\n\
\ ],\n [\n 331,\n 333\n ],\n [\n 337,\n 338\n ],\n [\n 388,\n 390\n ],\n [\n 398,\n 398\n ],\n [\n 0,\n 0\n ]\n ],\n \"tests/test_api_exceptions.py\": [\n [\n 34,\n 34\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"src/qwed_new/api/main.py::verify_math\",\n \"src/qwed_new/core/batch.py::_verify_item\"\n ],\n \"sites_in_set\": [\n \"src/qwed_new/core/batch.py::_verify_item\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"src/qwed_new/core/batch.py\",\n \"_verify_item\",\n 239\n ]\n ],\n \"kept\": [\n [\n \"src/qwed_new/core/batch.py\",\n \"_verify_item\",\n 239\n ]\n ],\n \"triage\": {\n \"src/qwed_new/core/batch.py::_verify_item\": \"parameter\"\n }\n}\n"
plane/workspaces/qwed-sympy-eval-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/reactpress-findall-column-sqli-inputs.yaml
+4 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,8 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"findAll\",\n \"line\": 108,\n \"path\": \"server/src/modules/article/article.service.ts\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"reactpress-findall-column-sqli\",\n \"sites\": [\n \"server/src/modules/article/article.service.ts::findAll\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.002177,\n \"recorded_usd\": 0.0193\n },\n \"family\": \"injection\",\n \"id\": \"reactpress-findall-column-sqli\",\n \"ranges\": {\n \"server/src/modules/article/article.service.ts\": [\n [\n 6,\n 6\n ],\n [\n 106,\n 110\n ],\n [\n 307,\n 308\n ]\n ],\n \"server/src/modules/comment/comment.controller.ts\": [\n [\n 20,\n 20\n ]\n ],\n \"server/src/modules/comment/comment.service.ts\": [\n [\n 5,\n 5\n ],\n [\n 6,\n 6\n ],\n [\n 45,\n 45\n ],\n [\n 103,\n 107\n ]\n ],\n \"server/src/modules/file/file.service.ts\": [\n [\n 7,\n 7\n ],\n [\n 49,\n 49\n ],\n [\n 75,\n\
\ 79\n ]\n ],\n \"server/src/modules/knowledge/knowledge.service.ts\": [\n [\n 6,\n 6\n ],\n [\n 137,\n 141\n ]\n ],\n \"server/src/modules/page/page.service.ts\": [\n [\n 6,\n 6\n ],\n [\n 49,\n 53\n ]\n ],\n \"server/src/starter.ts\": [\n [\n 22,\n 22\n ],\n [\n 34,\n 34\n ],\n [\n 39,\n 39\n ],\n [\n 43,\n 43\n ],\n [\n 49,\n 49\n ],\n [\n 51,\n 51\n ],\n [\n 54,\n 54\n ],\n [\n 71,\n 71\n ],\n [\n 76,\n 76\n ],\n [\n 80,\n 80\n ],\n [\n 82,\n 82\n ],\n [\n 85,\n 85\n ],\n [\n 90,\n 90\n ],\n [\n 103,\n 103\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"server/src/modules/article/article.service.ts::findAll\"\n ],\n \"sites_in_set\"\
: [\n \"server/src/modules/article/article.service.ts::findAll\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"server/src/modules/article/article.service.ts\",\n \"findAll\",\n 108\n ]\n ],\n \"kept\": [\n [\n \"server/src/modules/article/article.service.ts\",\n \"findAll\",\n 108\n ]\n ],\n \"triage\": {\n \"server/src/modules/article/article.service.ts::findAll\": \"parameter\"\n }\n}\n"
plane/workspaces/reactpress-findall-column-sqli-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/unleash-addon-webhook-ssrf-inputs.yaml
+5 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,9 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"fetchRetry\",\n \"line\": 64,\n \"path\": \"src/lib/addons/addon.ts\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"unleash-addon-webhook-ssrf\",\n \"sites\": [\n \"src/lib/addons/addon.ts::fetchRetry\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.000514,\n \"recorded_usd\": 0.0293\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"unleash-addon-webhook-ssrf\",\n \"ranges\": {\n \"src/lib/__snapshots__/create-config.test.ts.snap\": [\n [\n 16,\n 16\n ]\n ],\n \"src/lib/addons/addon.ts\": [\n [\n 1,\n 1\n ],\n [\n 10,\n 10\n ],\n [\n 24,\n 24\n ],\n [\n 31,\n 31\n ],\n [\n 34,\n 34\n ],\n [\n 60,\n 60\n ],\n [\n 64,\n 65\n ],\n [\n 66,\n 66\n ],\n [\n 68,\n 68\n ],\n [\n 70,\n 70\n ],\n [\n 72,\n 72\n ],\n [\n 74,\n 75\n ],\n [\n 77,\n 77\n ],\n [\n 98,\n 98\n\
\ ]\n ],\n \"src/lib/addons/datadog-definition.ts\": [\n [\n 31,\n 31\n ]\n ],\n \"src/lib/addons/datadog.test.ts\": [\n [\n 29,\n 29\n ]\n ],\n \"src/lib/addons/new-relic.test.ts\": [\n [\n 192,\n 192\n ]\n ],\n \"src/lib/addons/new-relic.ts\": [\n [\n 127,\n 127\n ]\n ],\n \"src/lib/addons/slack.test.ts\": [\n [\n 303,\n 304\n ]\n ],\n \"src/lib/addons/teams-workflow.test.ts\": [\n [\n 173,\n 173\n ]\n ],\n \"src/lib/addons/teams-workflow.ts\": [\n [\n 137,\n 137\n ]\n ],\n \"src/lib/addons/teams.test.ts\": [\n [\n 253,\n 253\n ]\n ],\n \"src/lib/addons/teams.ts\": [\n [\n 119,\n 119\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ],\n \"src/lib/create-config.ts\": [\n [\n 589,\n 589\n ],\n [\n 862,\n 862\n ]\n ],\n \"src/lib/features/metrics/integrations/integration-metrics.ts\": [\n [\n 15,\n 15\n ],\n [\n 44,\n 44\n\
\ ]\n ],\n \"src/lib/services/addon-service.test.ts\": [\n [\n 55,\n 55\n ],\n [\n 64,\n 64\n ],\n [\n 160,\n 160\n ],\n [\n 194,\n 194\n ],\n [\n 242,\n 242\n ],\n [\n 305,\n 305\n ],\n [\n 354,\n 354\n ],\n [\n 392,\n 392\n ],\n [\n 429,\n 429\n ],\n [\n 494,\n 494\n ]\n ],\n \"src/lib/services/addon-service.ts\": [\n [\n 33,\n 33\n ],\n [\n 60,\n 60\n ],\n [\n 70,\n 70\n ],\n [\n 73,\n 73\n ],\n [\n 85,\n 85\n ],\n [\n 94,\n 94\n ],\n [\n 239,\n 239\n ],\n [\n 273,\n 273\n ],\n [\n 362,\n 362\n ]\n ],\n \"src/lib/types/model.ts\": [\n [\n 474,\n 474\n ]\n ],\n \"src/lib/types/option.ts\": [\n [\n 189,\n 189\n ],\n [\n 320,\n 320\n ]\n ],\n \"src/test/e2e/api/admin/addon.e2e.test.ts\": [\n [\n 19,\n 19\n ]\n ],\n \"src/test/e2e/services/addon-service.e2e.test.ts\"\
: [\n [\n 25,\n 25\n ],\n [\n 41,\n 41\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"src/lib/addons/addon.ts::fetchRetry\"\n ],\n \"sites_in_set\": [\n \"src/lib/addons/addon.ts::fetchRetry\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"src/lib/addons/addon.ts\",\n \"fetchRetry\",\n 64\n ]\n ],\n \"kept\": [\n [\n \"src/lib/addons/addon.ts\",\n \"fetchRetry\",\n 64\n ]\n ],\n \"triage\": {\n \"src/lib/addons/addon.ts::fetchRetry\": \"parameter\"\n }\n}\n"
plane/workspaces/unleash-addon-webhook-ssrf-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/winml-cli-serve-cors-inputs.yaml
+4 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -13,6 +13,8 @@ spec:
'
- path: case.json
content: "{\n \"family\": \"config_secrets\",\n \"id\": \"winml-cli-serve-cors\",\n \"sites\": [\n \"src/winml/modelkit/serve/cli_api.py::<global>\",\n \"src/winml/modelkit/serve/app.py::create_app\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.0021,\n \"recorded_usd\": 0.0021\n },\n \"family\": \"config_secrets\",\n \"id\": \"winml-cli-serve-cors\",\n \"ranges\": {\n \".github/workflows/modelkit-ci.yml\": [\n [\n 38,\n 38\n ],\n [\n 53,\n 53\n ]\n ],\n \"src/winml/modelkit/commands/serve.py\": [\n [\n 137,\n 137\n ],\n [\n 174,\n 174\n ]\n ],\n \"src/winml/modelkit/loader/_autoconfig.py\": [\n [\n 156,\n 156\n ]\n ],\n \"src/winml/modelkit/loader/hf.py\": [\n [\n 204,\n 204\n ],\n [\n 206,\n 206\n ],\n [\n 0,\n 0\n ]\n ],\n \"src/winml/modelkit/serve/app.py\": [\n [\n 36,\n 37\n ],\n [\n 43,\n 43\n ],\n\
\ [\n 216,\n 216\n ],\n [\n 218,\n 222\n ],\n [\n 289,\n 289\n ],\n [\n 322,\n 322\n ],\n [\n 369,\n 369\n ],\n [\n 374,\n 374\n ],\n [\n 376,\n 376\n ],\n [\n 379,\n 379\n ],\n [\n 382,\n 382\n ],\n [\n 393,\n 393\n ]\n ],\n \"src/winml/modelkit/serve/cli_api.py\": [\n [\n 33,\n 33\n ],\n [\n 39,\n 39\n ],\n [\n 147,\n 147\n ],\n [\n 149,\n 153\n ],\n [\n 180,\n 180\n ],\n [\n 251,\n 251\n ],\n [\n 293,\n 293\n ],\n [\n 365,\n 369\n ]\n ],\n \"src/winml/modelkit/serve/manager.py\": [\n [\n 52,\n 52\n ],\n [\n 57,\n 57\n ],\n [\n 113,\n 113\n ],\n [\n 251,\n 251\n ],\n [\n 262,\n 262\n ],\n [\n 439,\n 439\n ],\n [\n 452,\n 452\n ]\n ],\n \"src/winml/modelkit/serve/static/index.html\": [\n [\n 5,\n 5\n ],\n [\n 300,\n\
\ 300\n ],\n [\n 920,\n 920\n ],\n [\n 945,\n 945\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"src/winml/modelkit/serve/cli_api.py::<global>\",\n \"src/winml/modelkit/serve/app.py::create_app\"\n ],\n \"sites_in_set\": [\n \"src/winml/modelkit/serve/cli_api.py::<global>\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"src/winml/modelkit/serve/cli_api.py\",\n \"<global>\",\n 148\n ]\n ],\n \"kept\": [],\n \"triage\": {\n \"src/winml/modelkit/serve/cli_api.py::<global>\": \"constant\"\n }\n}\n"
plane/workspaces/winml-cli-serve-cors-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/wp-directorist-avatar-ssrf-inputs.yaml
+3 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,7 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"directorist_rest_upload_image_from_url\",\n \"line\": 115,\n \"path\": \"includes/rest-api/functions.php\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-directorist-avatar-ssrf\",\n \"sites\": [\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.002616,\n \"recorded_usd\": 0.0352\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-directorist-avatar-ssrf\",\n \"ranges\": {\n \"config.php\": [\n [\n 4,\n 4\n ]\n ],\n \"directorist-base.php\": [\n [\n 6,\n 6\n ]\n ],\n \"includes/rest-api/Version1/class-orders-controller.php\": [\n [\n 84,\n 84\n ]\n ],\n \"includes/rest-api/Version1/class-users-controller.php\": [\n [\n 587,\n 589\n ],\n [\n 598,\n 598\n ],\n [\n 600,\n 601\n ],\n [\n 632,\n 632\n ]\n ],\n \"includes/rest-api/Version1/class-users-favorites-controller.php\": [\n [\n 131,\n 131\n ]\n ],\n \"includes/rest-api/functions.php\"\
: [\n [\n 105,\n 105\n ],\n [\n 114,\n 114\n ],\n [\n 115,\n 115\n ],\n [\n 181,\n 181\n ]\n ],\n \"readme.txt\": [\n [\n 8,\n 8\n ],\n [\n 302,\n 302\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\"\n ],\n \"sites_in_set\": [\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"includes/rest-api/functions.php\",\n \"directorist_rest_upload_image_from_url\",\n 115\n ]\n ],\n \"kept\": [\n [\n \"includes/rest-api/functions.php\",\n \"directorist_rest_upload_image_from_url\",\n 115\n ]\n ],\n \"triage\": {\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\": \"parameter\"\n }\n}\n"
plane/workspaces/wp-directorist-avatar-ssrf-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/wp-gopay-log-filter-sqli-inputs.yaml
+2 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,6 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"<global>\",\n \"line\": 35,\n \"path\": \"admin/views/log.php\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"injection\",\n \"id\": \"wp-gopay-log-filter-sqli\",\n \"sites\": [\n \"admin/views/log.php::<global>\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.001375,\n \"recorded_usd\": 0.0113\n },\n \"family\": \"injection\",\n \"id\": \"wp-gopay-log-filter-sqli\",\n \"ranges\": {\n \"admin/views/log.php\": [\n [\n 12,\n 13\n ],\n [\n 15,\n 15\n ],\n [\n 17,\n 22\n ],\n [\n 30,\n 30\n ],\n [\n 36,\n 41\n ],\n [\n 104,\n 104\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"admin/views/log.php::<global>\"\n ],\n \"sites_in_set\": [\n \"admin/views/log.php::<global>\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"admin/views/log.php\",\n \"<global>\",\n 35\n ]\n ],\n \"kept\": [\n [\n \"admin/views/log.php\",\n \"<global>\",\n 35\n ]\n ],\n \"triage\": {\n \"admin/views/log.php::<global>\": \"request\"\n }\n}\n"
plane/workspaces/wp-gopay-log-filter-sqli-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
plane/workspaces/wp-groundhogg-confirm-redirect-inputs.yaml
+2 -2
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
@@ -11,6 +11,6 @@ spec:
- path: functions.json
content: "[\n {\n \"function\": \"<global>\",\n \"line\": 904,\n \"path\": \"templates/preferences.php\"\n }\n]\n"
- path: case.json
content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-groundhogg-confirm-redirect\",\n \"sites\": [\n \"templates/preferences.php::<global>\"\n ]\n}\n"
content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.006524,\n \"recorded_usd\": 0.0585\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-groundhogg-confirm-redirect\",\n \"ranges\": {\n \"templates/preferences.php\": [\n [\n 904,\n 904\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"templates/preferences.php::<global>\"\n ],\n \"sites_in_set\": [\n \"templates/preferences.php::<global>\"\n ]\n}\n"
- path: triage.json
content: "{\n \"candidates\": [\n [\n \"templates/preferences.php\",\n \"<global>\",\n 904\n ]\n ],\n \"kept\": [\n [\n \"templates/preferences.php\",\n \"<global>\",\n 904\n ]\n ],\n \"triage\": {\n \"templates/preferences.php::<global>\": \"request\"\n }\n}\n"
plane/workspaces/wp-groundhogg-confirm-redirect-vulnerable.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate.
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate.
apiVersion: ax.io/v1alpha1
kind: Workspace
metadata:
src/openultrasast/cli.py
+18 -4
View File
@@ -256,6 +256,13 @@ def main(argv: list[str] | None = None) -> int:
plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
plane_ws.add_argument("--run", default="validation-46", help="with --validation-set: the Run name")
plane_ws.add_argument("--plane", type=Path, default=Path("plane"), help="with --validation-set: the manifest root")
plane_ws.add_argument(
"--repos", type=Path, default=Path.home() / ".cache" / "openultrasast" / "independent",
help="with --validation-set: one clone per case id, for the fix ranges agree scores by (evaluate.hunks)",
) # fmt: skip
plane_ws.add_argument(
"--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)"
)
args = parser.parse_args(argv)
# Diagnosis needs the stage costs, and nothing configures logging, so the default root level of
@@ -1324,11 +1331,18 @@ def _plane_increment(args: argparse.Namespace) -> int:
print("--validation-set needs --candidates and --triage", file=sys.stderr)
return 2
home = str(Path.home())
shown = [str(p).replace(home, "~", 1) for p in (args.population, args.validation_set, args.candidates, args.triage)]
command = f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]} --run {args.run}"
written = increment(
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command
paths = (args.population, args.validation_set, args.candidates, args.triage, args.repos)
shown = [str(p).replace(home, "~", 1) for p in paths]
command = (
f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]}"
f" --repos {shown[4]} --run {args.run}"
)
if args.runner_image is not None:
command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}"
written = increment(
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
repos=args.repos, runner_image=args.runner_image,
) # fmt: skip
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
return 0
src/openultrasast/plane/generate.py
+113 -9
View File
@@ -8,8 +8,11 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
``openultrasast.io/git-commits`` annotation, as :mod:`.workspaces` writes it);
- ``workspaces/<case>-inputs.yaml``: a files-only Workspace with the inputs no task produces -- the recorded
candidates (``candidates.json`` in the ``independent-v2-modelsinks`` shape for ``verify`` and ``agree``,
``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), the case's declared sites
(``case.json`` for ``agree``) and the recorded triage (``triage.json``, for the measurement);
``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), everything ``agree`` scores by
(``case.json``: the declared sites, the old-side fix ranges exactly as ``evaluate.hunks(case, "old")`` computes
them from ``--repos``, the declared sites among the set's candidates -- the reference's "of 20" -- and the
reference's cost: candidates before triage, recorded usd, and the triage share of it) and the recorded triage
(``triage.json``, for the measurement);
- ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
Workspaces and naming the files as env;
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets.
@@ -24,12 +27,21 @@ Candidates are the set's ``(path, function)`` pairs with the line of the case re
``benchmarks/independent/verify_batched.py`` does (sorted, the last line of a repeated pair wins), and filtered
by the recorded triage (the ``kept`` lists of the batched-check scans): ``verify`` has no triage stage, so the
hunts see exactly the candidates the reference hunts saw. Output is a pure function of the inputs.
The recorded triage cost is derived per file: a batched-check record's ``usd`` paid for the triage call and the
hunts, and its ``usage`` counts only the hunts (``verify_sinks._Recording`` slices the client's usage from the
first hunt on), so the triage share is ``usd`` minus that usage priced at the reference model's list price.
``--runner-image FILE`` (``~/.cache/ousast/ax-src/runner-image``, written by ``ops/ax/up.sh``) re-pins the
templates' ``image:`` to the digest in that file before generating, so a rebuilt image is one command.
"""
from __future__ import annotations
import json
import math
import re
import subprocess
import tomllib
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
@@ -38,13 +50,21 @@ from typing import Any
import yaml
from ..model.endpoint import price_of
from .budget import cost_of
from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests
from .reconciler import _ax_name, _doc
from .tasks.verify import MAX_STEPS, units_of
__all__ = ["CaseInputs", "increment", "read_cases", "render"]
__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
REFERENCE_USD_PER_CANDIDATE = 0.022 # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes
REFERENCE_MODEL = "deepseek-flash" # verify_batched.py's default (DEFAULT_DETECTOR_MODEL) in the batched check
TRIAGE_BASIS = (
"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage "
f"priced at {REFERENCE_MODEL} list prices; the triage call's own usage was not recorded"
)
RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')"
HEADROOM = 1.5
CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content retry
ATESPACE = "default"
@@ -62,6 +82,37 @@ class CaseInputs:
kept: tuple[Candidate, ...]
triage: Mapping[str, str]
recorded_usd: float
triage_usd: float = 0.0
ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None
def fix_ranges(repo: Path, vulnerable: str, fixed: str) -> dict[str, list[tuple[int, int]]]:
"""Changed line ranges per file on the vulnerable side of the fix: `evaluate.hunks(case, "old")`, line for
line (including its attribution of a hunk under ``--- /dev/null`` to the file before it)."""
if not (repo / ".git").exists() and not (repo / "HEAD").is_file():
raise ValueError(f"no repository at {repo}: the fix ranges cannot be computed")
diff = subprocess.run(["git", "-C", str(repo), "diff", "-U0", vulnerable, fixed], capture_output=True, text=True, check=True).stdout
ranges: dict[str, list[tuple[int, int]]] = {}
current = ""
for line in diff.splitlines():
if line.startswith("--- a/"):
current = line[6:]
elif line.startswith("@@") and current:
match = re.search(r"-(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?", line)
if match:
first, length = int(match.group(1)), int(match.group(2)) if match.group(2) is not None else 1
ranges.setdefault(current, []).append((first, first + max(length, 1) - 1))
if not ranges:
raise ValueError(f"{repo}: the diff {vulnerable[:12]}..{fixed[:12]} changed no line on the vulnerable side")
return ranges
def _triage_usd(rows: Sequence[Mapping[str, Any]]) -> float:
"""The triage share of the recorded spend: per file, usd minus the recorded hunt usage at list price."""
prices = price_of(REFERENCE_MODEL)
if prices is None:
raise ValueError(f"no recorded price for {REFERENCE_MODEL}")
return round(sum(max(float(r.get("usd") or 0.0) - cost_of(r.get("usage") or {}, prices), 0.0) for r in rows), 6)
def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
@@ -75,8 +126,11 @@ def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
return rows
def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path) -> list[CaseInputs]:
"""The set's cases in id order, each with its candidates, the triage-kept subset and the recorded cost."""
def read_cases(
validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None
) -> list[CaseInputs]:
"""The set's cases in id order, each with its candidates, the triage-kept subset, the recorded cost and its
triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges."""
subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8"))
records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]}
cases: list[CaseInputs] = []
@@ -104,6 +158,8 @@ def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, pop
kept=tuple(c for c in chosen if (c[0], c[1]) in kept_keys),
triage=dict(sorted(triage.items())),
recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6),
triage_usd=_triage_usd(rows),
ranges=None if repos is None else fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"]),
)
)
return cases
@@ -117,6 +173,23 @@ def _json(payload: object) -> str:
return json.dumps(payload, indent=1, sort_keys=True) + "\n"
def _case_record(item: CaseInputs) -> dict[str, Any]:
"""What `agree` scores by: sites, fix ranges (absent without ``--repos``), sites in the set, reference cost."""
case = item.case
sites = [str(s) for s in case.get("sites", [])]
in_set = {f"{p}::{fn}" for p, fn, _ in item.candidates}
record: dict[str, Any] = {"id": case["id"], "family": case["family"], "sites": sites}
if item.ranges is not None:
record["ranges"] = {path: [list(span) for span in spans] for path, spans in item.ranges.items()}
record["ranges_basis"] = RANGES_BASIS
record["sites_in_set"] = [s for s in sites if s in in_set]
record["cost"] = {
"candidates_before_triage": len(item.candidates), "recorded_usd": item.recorded_usd,
"recorded_triage_usd": item.triage_usd, "recorded_triage_basis": TRIAGE_BASIS,
} # fmt: skip
return record
def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
case = item.case
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
@@ -125,7 +198,7 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
files = {
"candidates.json": _json({"id": case["id"], "family": case["family"], "candidates": [list(c) for c in item.kept]}),
"functions.json": _json([{"path": p, "function": fn, "line": ln} for p, fn, ln in item.kept]),
"case.json": _json({"id": case["id"], "family": case["family"], "sites": list(case.get("sites", []))}),
"case.json": _json(_case_record(item)),
"triage.json": _json(
{"candidates": [list(c) for c in item.candidates], "kept": [list(c) for c in item.kept], "triage": dict(item.triage)}
),
@@ -207,12 +280,43 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name:
return out
_IMAGE_LINE = re.compile(r'^(\s*image:\s*)"?[^"\s]+"?\s*$', re.MULTILINE)
def repin_templates(plane: Path, image_file: Path) -> str:
"""Set every template's ``image:`` to the digest-pinned reference in ``image_file`` (as ``ops/ax/up.sh``
writes it); comments and layout are kept. Returns the reference."""
image = image_file.read_text(encoding="utf-8").strip()
if not re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", image):
raise ValueError(f"{image_file}: expected a digest-pinned image reference, got {image!r}")
for name in TEMPLATES:
path = plane / "tasks" / f"{name}.yaml"
text = path.read_text(encoding="utf-8")
pinned, count = _IMAGE_LINE.subn(lambda m: f'{m.group(1)}"{image}"', text)
if count != 1:
raise ValueError(f"{path}: {count} image lines, expected one")
path.write_text(pinned, encoding="utf-8")
return image
def increment(
population: Path, validation_set: Path, candidates_dir: Path, triage_dir: Path, *, plane: Path, run_name: str, command: str
population: Path,
validation_set: Path,
candidates_dir: Path,
triage_dir: Path,
*,
plane: Path,
run_name: str,
command: str,
repos: Path | None = None,
runner_image: Path | None = None,
) -> list[Path]:
"""Write the generated manifests under ``plane`` from the templates in ``plane/tasks``; returns the paths."""
"""Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when
``runner_image`` is given); returns the paths."""
if runner_image is not None:
repin_templates(plane, runner_image)
loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
files = render(read_cases(validation_set, candidates_dir, triage_dir, population), loaded.tasks, run_name, command)
files = render(read_cases(validation_set, candidates_dir, triage_dir, population, repos), loaded.tasks, run_name, command)
written: list[Path] = []
for relative, text in files.items():
path = plane / relative
src/openultrasast/plane/tasks/agree.py
+70 -44
View File
@@ -3,31 +3,65 @@
The rule is `verify_batched.agreement`: a candidate is reported when every pass flagged it; a candidate flagged
by one pass is `disputed`. Beside the verdicts go the measurement's numbers: per-candidate cost (a hunt's usd
split evenly over the candidates it asked about, summed over the passes), tool turns per hunt, and -- when the
case's declared sites are given -- the site match by `benchmarks/independent/evaluate.matches_v2`, the rule
protocol v2 scores by, imported from the benchmark tree rather than copied.
case's declared sites and fix ranges are given -- the site match by :func:`matches_declared`, which is
`benchmarks/independent/evaluate.matches_v2` (the rule protocol v2 and `score_batched.py` score by) reproduced
here so the runner image needs no benchmark tree; `tests/test_plane_scoring_parity.py` proves identical verdicts
on the recorded reference outputs. The ranges are the reference's own (`evaluate.hunks(case, "old")`), resolved
by the generator into `case.json`, and sites are anchored to the candidate as `score_batched.anchored` does.
Cost is reported twice: `cost_per_candidate` is the plane's spend alone; `cost_per_candidate_with_recorded_triage`
adds the recorded triage cost of the case (`case.json` `cost.recorded_triage_usd`), because the reference's
$0.022 per candidate for two passes paid for triage and the plane applies the recorded triage instead. Both
divide by the candidates before triage when `case.json` gives that count, as the reference did.
Env contract: `OUSAST_INPUT_PASS_A`, `OUSAST_INPUT_PASS_B` (a verify output directory or its `units.jsonl`),
`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (a JSON case record with `family`
and `sites` in the `population-v2.toml` shape, or that TOML itself with `OUSAST_CASE_ID` naming the case) and
`OUSAST_BENCHMARKS_DIR` (where `independent/evaluate.py` lives when the package is not run from the repository).
Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished.
`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (the generator's `case.json`:
`family`, `sites`, `ranges`, `sites_in_set`, `cost`; or a `population-v2.toml`-shaped TOML with `OUSAST_CASE_ID`
naming the case). Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished.
"""
from __future__ import annotations
import importlib.util
import json
import os
import sys
import tomllib
import traceback
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from .verify import EXIT_CODES, USAGE_FIELDS, load_candidates, read_units
Matcher = Callable[[dict[str, Any], dict[str, Any], dict[str, list[tuple[int, int]]]], bool]
LINE_WINDOW = 10 # evaluate.LINE_WINDOW: a finding within 10 lines of a changed range matches
def matches_declared(finding: Mapping[str, Any], case: Mapping[str, Any], ranges: Mapping[str, Sequence[Sequence[int]]]) -> bool:
"""`evaluate.matches_v2`: the finding's family is the case's, and its `path:line:function` site is a declared
`path::function`, or lies in a file the fix changed and names a declared site's function or is within
`LINE_WINDOW` lines of a changed range (`ranges`: the old side of the fix diff, per file)."""
path, _, rest = str(finding["site"]).partition(":")
line_text, _, function = rest.partition(":")
if finding.get("family") != case["family"]:
return False
sites = {str(s) for s in case.get("sites", [])}
if f"{path}::{function}" in sites:
return True
if path not in ranges:
return False
if function in {site.partition("::")[2] for site in sites}:
return True
line = int(line_text) if line_text.isdigit() else -1
return any(int(first) - LINE_WINDOW <= line <= int(last) + LINE_WINDOW for first, last in ranges[path])
def anchor_site(candidate: str, site: str) -> str:
"""`score_batched.anchored`: the site is the candidate's file and function; the reported line is kept only
when the site is in the candidate's file (otherwise 0)."""
path, _, function = candidate.partition("::")
site_path, _, rest = site.partition(":")
line = rest.split(":")[0] if site_path == path else "0"
return f"{path}:{line}:{function}"
def agreement(passes: Sequence[Sequence[Mapping[str, Any]]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
@@ -51,7 +85,8 @@ def load_pass(source: Path) -> list[dict[str, Any]]:
def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None:
"""A case record with `family`, `sites` and optional `ranges` (`{path: [[first, last], ...]}`)."""
"""A case record with `family`, `sites` and `ranges` (`{path: [[first, last], ...]}`); without `ranges`
site matching is reported as not assessed."""
if path is None:
return None
if path.suffix == ".toml":
@@ -66,24 +101,6 @@ def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None:
return record
def load_matcher(benchmarks_dir: Path | None = None) -> Matcher | None:
"""`evaluate.matches_v2` from the benchmark tree, loaded by file so the package never imports it by name;
None when the tree is not present (site matching is then reported as not assessed, never as a miss)."""
roots = [benchmarks_dir] if benchmarks_dir is not None else []
roots += [p / "benchmarks" for p in (Path.cwd(), *Path(__file__).resolve().parents)]
for root in roots:
script = root / "independent" / "evaluate.py"
if script.is_file():
spec = importlib.util.spec_from_file_location("ousast_benchmark_evaluate", script)
if spec is None or spec.loader is None:
continue
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
matcher: Matcher = module.matches_v2
return matcher
return None
def _per_candidate_cost(rows: Sequence[Mapping[str, Any]]) -> tuple[dict[str, float | None], dict[str, int]]:
"""A hunt's usd split evenly over its candidates; its turns and calls counted once per candidate."""
usd: dict[str, float | None] = {}
@@ -101,6 +118,10 @@ def _mean(values: Sequence[float]) -> float | None:
return round(sum(values) / len(values), 3) if values else None
def _per_candidate(total: float | None, count: int) -> float | None:
return None if total is None or not count else round(total / count, 6)
def run(
output_dir: Path,
candidates: object,
@@ -108,7 +129,6 @@ def run(
pass_b: Sequence[Mapping[str, Any]],
*,
sites: Mapping[str, Any] | None = None,
matcher: Matcher | None = None,
) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
family, unique = load_candidates(candidates)
@@ -121,18 +141,17 @@ def run(
case = dict(sites) if sites else None
if case is not None and "family" not in case and family:
case["family"] = family
ranges = {p: [(int(a), int(b)) for a, b in spans] for p, spans in (case or {}).get("ranges", {}).items()}
match_state = (
"not assessed: no sites given" if case is None else ("not assessed: evaluate.py not found" if matcher is None else "assessed")
)
ranges = (case or {}).get("ranges")
assessed = case is not None and isinstance(ranges, Mapping)
match_state = "assessed" if assessed else ("not assessed: no sites given" if case is None else "not assessed: no fix ranges given")
rows: list[dict[str, Any]] = []
for name in names:
in_a, in_b = (name in flagged[0] if name in asked[0] else None), (name in flagged[1] if name in asked[1] else None)
finding = flagged[0].get(name) or flagged[1].get(name)
site = finding["site"] if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}"
site = anchor_site(name, finding["site"]) if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}"
site_match = None
if case is not None and matcher is not None:
site_match = matcher({"site": site, "family": (finding or {}).get("family", family)}, case, ranges)
if assessed and case is not None and isinstance(ranges, Mapping):
site_match = matches_declared({"site": site, "family": (finding or {}).get("family", family)}, case, ranges)
cost = usd_by.get(name)
rows.append({
"candidate": name, "site": site, "a": in_a, "b": in_b, "agreed": bool(in_a and in_b), "disputed": bool(in_a) != bool(in_b),
@@ -142,12 +161,20 @@ def run(
declared = [r for r in rows if r["site_match"]]
hunts = [*pass_a, *pass_b]
usd_total = None if any(r.get("usd") is None for r in hunts) else round(sum(float(r["usd"]) for r in hunts), 6)
reference: Mapping[str, Any] = (case or {}).get("cost") or {}
before = int(reference.get("candidates_before_triage") or len(names))
triage_usd = float(reference["recorded_triage_usd"]) if reference.get("recorded_triage_usd") is not None else None
with_triage = None if usd_total is None or triage_usd is None else round(usd_total + triage_usd, 6)
metrics = {
"candidates": len(names), "asked_in_both": len(names) - len(unasked), "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]),
"agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts),
"declared_sites": len(set((case or {}).get("sites", []))), "declared_sites_matched": len(declared),
"declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]), "site_matching": match_state,
"usd_total": usd_total, "cost_per_candidate": (None if usd_total is None or not names else round(usd_total / len(names), 6)),
"candidates": len(names), "candidates_before_triage": before, "asked_in_both": len(names) - len(unasked),
"flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]), "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts),
"declared_sites": len(set((case or {}).get("sites", []))),
"declared_sites_in_set": (len(case["sites_in_set"]) if case and "sites_in_set" in case else None),
"declared_sites_matched": len(declared), "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]),
"site_matching": match_state,
"usd_total": usd_total, "cost_per_candidate": _per_candidate(usd_total, before),
"recorded_triage_usd": triage_usd, "recorded_triage_basis": reference.get("recorded_triage_basis"),
"usd_total_with_recorded_triage": with_triage, "cost_per_candidate_with_recorded_triage": _per_candidate(with_triage, before),
"turns_per_hunt": _mean([float(r.get("turns") or 0) for r in hunts]),
"calls_per_hunt": _mean([float((r.get("usage") or {}).get("calls") or 0) for r in hunts]),
} # fmt: skip
@@ -174,8 +201,7 @@ def main() -> int:
candidates = json.loads(Path(os.environ["OUSAST_INPUT_CANDIDATES"]).read_text())
sites_path = Path(os.environ["OUSAST_INPUT_SITES"]) if os.environ.get("OUSAST_INPUT_SITES") else None
sites = load_sites(sites_path, os.environ.get("OUSAST_CASE_ID"))
benchmarks = Path(os.environ["OUSAST_BENCHMARKS_DIR"]) if os.environ.get("OUSAST_BENCHMARKS_DIR") else None
summary = run(output_dir, candidates, pass_a, pass_b, sites=sites, matcher=load_matcher(benchmarks) if sites else None)
summary = run(output_dir, candidates, pass_a, pass_b, sites=sites)
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
summary = {
"status": "failed", "units_done": 0, "units_total": 0, "usd": 0, "calls": 0, "usage": dict.fromkeys(USAGE_FIELDS, 0),
@@ -188,7 +214,7 @@ def main() -> int:
return EXIT_CODES.get(str(summary["status"]), 2)
__all__ = ["agreement", "load_matcher", "load_pass", "load_sites", "main", "run"]
__all__ = ["LINE_WINDOW", "agreement", "anchor_site", "load_pass", "load_sites", "main", "matches_declared", "run"]
if __name__ == "__main__":
tests/test_plane_increment.py
+65 -4
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import json
import re
import shutil
import subprocess
import tomllib
from pathlib import Path
@@ -15,13 +16,14 @@ from test_plane_reconciler import AX_FIELDS, non_ax_fields
from openultrasast.model.endpoint import price_of
from openultrasast.plane import reconciler
from openultrasast.plane.budget import prices_from
from openultrasast.plane.generate import TEMPLATES, increment, read_cases, render
from openultrasast.plane.generate import TEMPLATES, fix_ranges, increment, read_cases, render, repin_templates
from openultrasast.plane.manifests import load_manifests
PLANE = Path("plane")
RUN = PLANE / "runs" / "validation-46.yaml"
POPULATION = Path("benchmarks/independent/population-v2.toml")
RESULTS = Path.home() / "ousast-results"
REPOS = Path.home() / ".cache" / "openultrasast" / "independent"
PIN = "0123456789abcdef0123456789abcdef01234567"
@@ -80,14 +82,30 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
def test_inputs_carry_the_validation_set() -> None:
manifests = load_manifests(sorted((PLANE / "workspaces").glob("*-inputs.yaml")))
assert len(manifests.workspaces) == 15
candidates = kept = 0
candidates = kept = in_set = before = 0
usd = triage = 0.0
for workspace in manifests.workspaces.values():
files = {f.path: json.loads(f.content) for f in workspace.files}
assert set(files) == {"candidates.json", "functions.json", "case.json", "triage.json"}
candidates += len(files["triage.json"]["candidates"])
kept += len(files["candidates.json"]["candidates"])
assert [[r["path"], r["function"], r["line"]] for r in files["functions.json"]] == files["candidates.json"]["candidates"]
assert (candidates, kept) == (46, 43)
case = files["case.json"]
assert case["ranges"] and all(first <= last for spans in case["ranges"].values() for first, last in spans), case["id"]
assert set(case["sites_in_set"]) <= set(case["sites"])
in_set += len(case["sites_in_set"])
before += case["cost"]["candidates_before_triage"]
usd += case["cost"]["recorded_usd"]
triage += case["cost"]["recorded_triage_usd"]
assert 0 < case["cost"]["recorded_triage_usd"] <= case["cost"]["recorded_usd"], case["id"]
assert (candidates, kept, in_set, before) == (46, 43, 20, 46)
# the reference's $1.01 for 46 candidates, two passes, triage included; triage was about 9% of it
assert round(usd / before, 3) == 0.022 and 0.05 < triage / usd < 0.15
def test_templates_are_pinned_to_one_runner_image() -> None:
images = {task.image for task in load_manifests(sorted((PLANE / "tasks").glob("*.yaml"))).tasks.values()}
assert len(images) == 1 and re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", images.pop() or "")
def _committed_command() -> str:
@@ -97,7 +115,9 @@ def _committed_command() -> str:
return found.group(1)
@pytest.mark.skipif(not (RESULTS / "independent-v2-batched-check" / "scans").is_dir(), reason="recorded results not on this host")
@pytest.mark.skipif(
not (RESULTS / "independent-v2-batched-check" / "scans").is_dir() or not REPOS.is_dir(), reason="recorded results not on this host"
)
def test_committed_manifests_match_the_generator() -> None:
templates = load_manifests([PLANE / "tasks" / f"{name}.yaml" for name in TEMPLATES]).tasks
cases = read_cases(
@@ -105,6 +125,7 @@ def test_committed_manifests_match_the_generator() -> None:
RESULTS / "independent-v2-modelsinks",
RESULTS / "independent-v2-batched-check" / "scans",
POPULATION,
REPOS,
)
for relative, text in render(cases, templates, "validation-46", _committed_command()).items():
assert (PLANE / relative).read_text() == text, relative
@@ -172,3 +193,43 @@ def test_generator_refuses_a_set_the_recorded_scans_did_not_ask(tmp_path: Path)
(paths["scans"] / "alpha--vulnerable_a.jsonl").unlink()
with pytest.raises(ValueError, match="no recorded triage"):
_generate(paths, paths["plane"])
def _git(repo: Path, *args: str) -> str:
done = subprocess.run(
["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t", *args], capture_output=True, text=True, check=True
)
return done.stdout.strip()
def test_fix_ranges_are_the_old_side_of_the_fix_diff(tmp_path: Path) -> None:
repo = tmp_path / "repo"
repo.mkdir()
_git(repo, "init", "-q")
(repo / "a.py").write_text("".join(f"line {i}\n" for i in range(1, 31)))
_git(repo, "add", "-A")
_git(repo, "commit", "-qm", "vulnerable")
vulnerable = _git(repo, "rev-parse", "HEAD")
(repo / "a.py").write_text("".join(f"line {i}\n" if i not in (3, 4, 20) else "fixed\n" for i in range(1, 31)) + "added\n")
_git(repo, "commit", "-qam", "fixed")
assert fix_ranges(repo, vulnerable, _git(repo, "rev-parse", "HEAD")) == {"a.py": [(3, 4), (20, 20), (30, 30)]}
with pytest.raises(ValueError, match="no repository"):
fix_ranges(tmp_path / "absent", vulnerable, vulnerable)
def test_case_record_carries_sites_in_set_and_cost_and_the_image_is_repinned(tmp_path: Path) -> None:
paths = _fixture(tmp_path)
digest = "localhost:5001/ousast-runner@sha256:" + "ab" * 32
(tmp_path / "runner-image").write_text(digest + "\n")
increment(
paths["population"], paths["set"], paths["candidates"], paths["scans"], plane=paths["plane"], run_name="check",
command="ousast plane test", runner_image=tmp_path / "runner-image",
) # fmt: skip
manifests = load_manifests(sorted(paths["plane"].rglob("*.yaml")))
assert {t.image for t in manifests.tasks.values()} == {digest}
case = json.loads(next(f.content for f in manifests.workspaces["alpha-inputs"].files if f.path == "case.json"))
assert "ranges" not in case and case["sites_in_set"] == ["a.py::run"] # no --repos: agree reports not assessed
assert case["cost"]["candidates_before_triage"] == 2 and case["cost"]["recorded_triage_usd"] == 0.5 # no hunt usage recorded
(tmp_path / "runner-image").write_text("localhost:5001/ousast-runner:dev\n")
with pytest.raises(ValueError, match="digest-pinned"):
repin_templates(paths["plane"], tmp_path / "runner-image")
tests/test_plane_scoring_parity.py
+144
View File
@@ -0,0 +1,144 @@
"""`agree` scores exactly as the reference measurement did (ai-service-plane Requirement 6).
The reference is `benchmarks/independent/score_batched.py` over the batched-check scans: sites anchored by
`score_batched.anchored`, matched by `evaluate.matches_v2` against `evaluate.hunks(case, "old")`. The plane cannot
ship the benchmark tree, so the generator resolves the ranges into `case.json` and `agree.matches_declared`
reproduces the rule. These tests run both on the recorded reference outputs of the 15 validation cases and
require identical verdicts, identical ranges, and the reference's own numbers (16 of 20, $0.022) back from `agree`.
"""
from __future__ import annotations
import importlib.util
import json
import sys
import tomllib
from pathlib import Path
from types import ModuleType
from typing import Any
import pytest
from openultrasast.plane.manifests import load_manifests
from openultrasast.plane.tasks import agree
BENCH = Path("benchmarks/independent")
POPULATION = BENCH / "population-v2.toml"
SCANS = Path.home() / "ousast-results" / "independent-v2-batched-check" / "scans"
REPOS = Path.home() / ".cache" / "openultrasast" / "independent"
INPUTS = sorted(Path("plane/workspaces").glob("*-inputs.yaml"))
pytestmark = pytest.mark.skipif(
not SCANS.is_dir() or not REPOS.is_dir(), reason="recorded batched-check scans or case clones not on this host"
)
def _load(name: str) -> ModuleType:
sys.path.insert(0, str(BENCH.resolve())) # score_batched imports evaluate by name
try:
spec = importlib.util.spec_from_file_location(f"parity_{name}", BENCH / f"{name}.py")
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
finally:
sys.path.remove(str(BENCH.resolve()))
@pytest.fixture(scope="module")
def reference() -> tuple[ModuleType, ModuleType]:
evaluate, score_batched = _load("evaluate"), _load("score_batched")
evaluate.use(POPULATION.resolve())
return evaluate, score_batched
def _inputs() -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for workspace in load_manifests(INPUTS).workspaces.values():
files = {f.path: json.loads(f.content) for f in workspace.files}
out[files["case.json"]["id"]] = files
assert len(out) == 15, sorted(out)
return out
def _records() -> dict[str, dict[str, Any]]:
return {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]}
def test_committed_ranges_are_the_reference_hunks(reference: tuple[ModuleType, ModuleType]) -> None:
evaluate, _ = reference
records = _records()
for case_id, files in _inputs().items():
hunks = {path: [list(span) for span in spans] for path, spans in evaluate.hunks(records[case_id], "old").items()}
assert files["case.json"]["ranges"] == hunks, case_id
def test_matching_verdicts_are_identical_on_the_recorded_outputs(reference: tuple[ModuleType, ModuleType]) -> None:
evaluate, score_batched = reference
records = _records()
compared = declared = matched = 0
for case_id, files in _inputs().items():
record, mine = records[case_id], files["case.json"]
old = evaluate.hunks(record, "old")
recorded = json.loads((SCANS / f"{case_id}--vulnerable_a.json").read_text())
raw = [dict(f) for key in ("findings", "disputed") for f in recorded.get(key, [])]
anchored = score_batched.anchored(json.loads(json.dumps({"findings": raw})))["findings"]
assert [agree.anchor_site(f["candidate"], f["site"]) for f in raw] == [f["site"] for f in anchored], case_id
family = record["family"]
probes = [{"site": f["site"], "family": f["family"]} for f in anchored]
probes += [{"site": f"{p}:{ln}:{fn}", "family": family} for p, fn, ln in files["triage.json"]["candidates"]]
sites = [{"site": f"{s.partition('::')[0]}:0:{s.partition('::')[2]}", "family": family} for s in record.get("sites", [])]
declared += len(sites)
probes += sites
probes += [dict(p, family="not-" + family) for p in probes]
for probe in probes:
theirs = evaluate.matches_v2(probe, record, old)
assert agree.matches_declared(probe, mine, mine["ranges"]) == theirs, (case_id, probe)
compared += 1
matched += theirs
assert declared == 22 and compared > 2 * (declared + 46)
assert matched > 0
def _units(case_id: str, kept: list[list[Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], float]:
"""The recorded per-file hunts as two `verify` passes: a hunt's usd is the file's spend minus its triage."""
from openultrasast.model.endpoint import price_of
from openultrasast.plane.budget import cost_of
prices = price_of("deepseek-flash")
assert prices is not None
passes: tuple[list[dict[str, Any]], list[dict[str, Any]]] = ([], [])
triage = 0.0
for text in (SCANS / f"{case_id}--vulnerable_a.jsonl").read_text().splitlines():
row = json.loads(text)
hunts = cost_of(row.get("usage") or {}, prices)
triage += max(float(row["usd"]) - hunts, 0.0)
asked = [c for c in kept if c[0] == row["path"]]
if not asked:
continue
for index, flagged in enumerate(row["passes"]):
passes[index].append({"path": row["path"], "candidates": asked, "flagged": flagged, "usd": hunts / 2, "turns": 0})
return passes[0], passes[1], triage
def test_agree_on_the_reference_outputs_reproduces_the_reference_numbers(tmp_path: Path) -> None:
score = json.loads((SCANS.parent / "score.json").read_text())
detected = {c["id"]: len(c["detected_sites"]) for c in score["cases"] if "detected_sites" in c and c["detected_sites"] is not None}
agreed = in_set = before = 0
spend = with_triage = 0.0
for case_id, files in _inputs().items():
case = files["case.json"]
pass_a, pass_b, triage = _units(case_id, files["candidates.json"]["candidates"])
assert case["cost"]["recorded_triage_usd"] == pytest.approx(triage, abs=1e-6)
summary = agree.run(tmp_path / case_id, files["candidates.json"], pass_a, pass_b, sites=case)
metrics = summary["metrics"]
assert metrics["site_matching"] == "assessed"
assert metrics["declared_sites_matched_agreed"] == detected[case_id], case_id
agreed += metrics["declared_sites_matched_agreed"]
in_set += metrics["declared_sites_in_set"]
before += metrics["candidates_before_triage"]
spend += metrics["usd_total"]
with_triage += metrics["usd_total_with_recorded_triage"]
assert (agreed, in_set, before) == (16, 20, 46)
assert with_triage == pytest.approx(1.0123, abs=1e-3) # the recorded spend: the reference's $1.01
assert round(with_triage / before, 3) == 0.022 and spend < with_triage
tests/test_plane_verify.py
+32 -3
View File
@@ -230,8 +230,8 @@ def _two_passes(tmp_path: Path) -> tuple[Path, Path, Path]:
def test_agree_over_two_scripted_passes_computes_agreement_and_metrics(tmp_path: Path) -> None:
_, a, b = _two_passes(tmp_path)
out = tmp_path / "agree"
sites = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"]}
summary = agree.run(out, CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=sites, matcher=agree.load_matcher())
sites = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"], "ranges": {"app.py": [[40, 41]]}}
summary = agree.run(out, CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=sites)
assert summary["status"] == "done" and summary["usd"] == 0 and summary["calls"] == 0 and summary["model"] is None
report = json.loads((out / "agreed.json").read_text())
by_name = {r["candidate"]: r for r in report["candidates"]}
@@ -255,6 +255,34 @@ def test_agree_over_two_scripted_passes_computes_agreement_and_metrics(tmp_path:
assert by_name["app.py::run"]["usd"] == pytest.approx(2 * per_call, abs=1e-6)
def test_agree_reports_cost_with_the_recorded_triage_and_needs_fix_ranges_to_assess_sites(tmp_path: Path) -> None:
_, a, b = _two_passes(tmp_path)
cost = {"candidates_before_triage": 4, "recorded_usd": 1.0, "recorded_triage_usd": 0.01, "recorded_triage_basis": "derived"}
sites = {"id": "case", "family": "injection", "sites": ["app.py::run"], "sites_in_set": ["app.py::run"], "cost": cost}
metrics = agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=sites)["metrics"]
spend = metrics["usd_total"]
assert metrics["candidates"] == 3 and metrics["candidates_before_triage"] == 4 and metrics["declared_sites_in_set"] == 1
assert metrics["cost_per_candidate"] == pytest.approx(spend / 4, abs=1e-6)
assert metrics["usd_total_with_recorded_triage"] == pytest.approx(spend + 0.01, abs=1e-6)
assert metrics["cost_per_candidate_with_recorded_triage"] == pytest.approx((spend + 0.01) / 4, abs=1e-6)
# without the fix ranges a miss cannot be told from a match: not assessed, never a silent zero
assert metrics["site_matching"] == "not assessed: no fix ranges given" and metrics["declared_sites_matched"] == 0
def test_matches_declared_is_the_protocol_v2_rule() -> None:
case = {"family": "injection", "sites": ["a.py::run", "b.py::<global>"]}
ranges = {"a.py": [[50, 52]], "c.py": [[5, 5]]}
hit = lambda site, family="injection": agree.matches_declared({"site": site, "family": family}, case, ranges) # noqa: E731
assert hit("a.py:0:run") and hit("b.py:0:<global>") # a declared path::function, whether or not the fix touched it
assert not hit("a.py:0:run", "ssrf") # another family never matches
assert hit("a.py:62:helper") and not hit("a.py:63:helper") and hit("a.py:40:x") and not hit("a.py:39:x") # 10-line window
assert hit("c.py:900:run") and not hit("d.py:1:run") # a declared function name in a changed file only
assert (
not hit("a.py:x:helper") and hit("c.py:x:helper") and agree.anchor_site("a.py::run", "other.py:7:run") == "a.py:0:run"
) # a non-numeric line is -1, as in the reference
assert agree.anchor_site("a.py::run", "a.py:7:renamed") == "a.py:7:run"
def test_agree_without_sites_leaves_site_matching_unassessed_and_flags_unasked_candidates(tmp_path: Path) -> None:
_, a, b = _two_passes(tmp_path)
rows_b = [r for r in agree.load_pass(b) if r["path"] != "db.py"]
@@ -269,7 +297,8 @@ def test_agree_main_reads_the_env_contract(tmp_path: Path, monkeypatch: pytest.M
_, a, b = _two_passes(tmp_path)
(tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES))
(tmp_path / "case.toml").write_text(
'[[case]]\nid = "case"\nfamily = "injection"\nsites = ["app.py::run"]\n\n[[case]]\nid = "x"\nfamily = "injection"\nsites = []\n'
'[[case]]\nid = "case"\nfamily = "injection"\nsites = ["app.py::run"]\nranges = { "app.py" = [[40, 41]] }\n\n'
'[[case]]\nid = "x"\nfamily = "injection"\nsites = []\n'
)
env = {
"OUSAST_OUTPUT_DIR": str(tmp_path / "agree"), "OUSAST_INPUT_PASS_A": str(a), "OUSAST_INPUT_PASS_B": str(b / "units.jsonl"),