diff --git a/.kiro/specs/ai-service-plane/design.md b/.kiro/specs/ai-service-plane/design.md index 3e09905..986b085 100644 --- a/.kiro/specs/ai-service-plane/design.md +++ b/.kiro/specs/ai-service-plane/design.md @@ -166,6 +166,17 @@ artifacts for the increment; the `classify` and `triage` tasks are later specs. Reads the pass outputs, emits `agreed.json` and `disputed.json` by the same rule as `verify_batched.agreement`, plus the per-candidate cost, turns and the declared-site match (`evaluate.matches_v2`) for the measurement. +The rule is reproduced in the package (`agree.matches_declared`, parity-tested against `matches_v2` on the recorded +reference outputs) and the generator writes the fix ranges it matches against into each case's `case.json`, so +the runner image carries no benchmark tree. + +**Measurement.** Requirement 6.2's cost is judged on `cost_per_candidate_with_recorded_triage`: the plane's two +verify passes plus the recorded triage cost of the case, over the candidates before triage, summed across the run. +The reference's $0.022 (`verifier-batched-check-2026-09-29.json`) paid for per-file triage and the hunts while the +plane applies the recorded triage without a model call, so `cost_per_candidate` (plane spend alone) is reported but +not judged; the triage share is derived per file from the recorded spend minus the priced hunt usage (about $0.087 +of the $1.01) and labelled as derived in `case.json`. Sites are judged as `declared_sites_matched_agreed` against +`declared_sites_in_set`, the reference's 16 of 20. ## Data Models diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 8f2cc7c..d3f9cf0 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -403,6 +403,7 @@ "model.specs", "model.taint", "plane.budget", + "plane.generate", "plane.tasks.repo_facts", "preprocess", "push.assistance", @@ -485,6 +486,7 @@ "importers": [ "cli", "plane.budget", + "plane.generate", "push.assistance", "tool_hunter" ] @@ -672,6 +674,7 @@ "classification": "load_bearing", "reason": "ai-service-plane Req 2.3/2.4: metered chat client with usd and calls ceilings; task integration in tasks 3, 6", "importers": [ + "plane.generate", "plane.tasks.verify" ] }, diff --git a/plane/runs/validation-46.yaml b/plane/runs/validation-46.yaml index 9d4116f..8fe8800 100644 --- a/plane/runs/validation-46.yaml +++ b/plane/runs/validation-46.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. # 15 cases, 46 candidates (43 kept by the recorded triage); usd ceiling 2.02. apiVersion: openultrasast.io/v1alpha1 kind: Run diff --git a/plane/tasks/agree.yaml b/plane/tasks/agree.yaml index 2f3d0f7..010b303 100644 --- a/plane/tasks/agree.yaml +++ b/plane/tasks/agree.yaml @@ -7,7 +7,7 @@ metadata: name: agree atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3" + image: "localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4" command: ["agree"] resources: requests: diff --git a/plane/tasks/repo-facts.yaml b/plane/tasks/repo-facts.yaml index fc189c0..d629b10 100644 --- a/plane/tasks/repo-facts.yaml +++ b/plane/tasks/repo-facts.yaml @@ -8,7 +8,7 @@ metadata: name: repo-facts atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3" + image: "localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4" command: ["repo-facts"] resources: requests: diff --git a/plane/tasks/validation-46.yaml b/plane/tasks/validation-46.yaml index 1579621..5de0bb7 100644 --- a/plane/tasks/validation-46.yaml +++ b/plane/tasks/validation-46.yaml @@ -1,11 +1,11 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Task metadata: name: repo-facts-budibase-mongo-template-nosqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -34,7 +34,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -65,7 +65,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -94,7 +94,7 @@ metadata: name: agree-budibase-mongo-template-nosqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -121,7 +121,7 @@ metadata: name: repo-facts-cct-studio-server-cmdi atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -150,7 +150,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -181,7 +181,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -210,7 +210,7 @@ metadata: name: agree-cct-studio-server-cmdi atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -237,7 +237,7 @@ metadata: name: repo-facts-contextforge-prompt-ssti atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -266,7 +266,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -297,7 +297,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -326,7 +326,7 @@ metadata: name: agree-contextforge-prompt-ssti atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -353,7 +353,7 @@ metadata: name: repo-facts-froxlor-admin-ip-second-order-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -382,7 +382,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -413,7 +413,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -442,7 +442,7 @@ metadata: name: agree-froxlor-admin-ip-second-order-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -469,7 +469,7 @@ metadata: name: repo-facts-fuxa-tdengine-escape-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -498,7 +498,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -529,7 +529,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -558,7 +558,7 @@ metadata: name: agree-fuxa-tdengine-escape-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -585,7 +585,7 @@ metadata: name: repo-facts-lmdeploy-media-url-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -614,7 +614,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -645,7 +645,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -674,7 +674,7 @@ metadata: name: agree-lmdeploy-media-url-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -701,7 +701,7 @@ metadata: name: repo-facts-medplum-external-auth-open-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -730,7 +730,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -761,7 +761,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -790,7 +790,7 @@ metadata: name: agree-medplum-external-auth-open-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -817,7 +817,7 @@ metadata: name: repo-facts-pgadmin-maintenance-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -846,7 +846,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -877,7 +877,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -906,7 +906,7 @@ metadata: name: agree-pgadmin-maintenance-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -933,7 +933,7 @@ metadata: name: repo-facts-qwed-sympy-eval atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -962,7 +962,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -993,7 +993,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1022,7 +1022,7 @@ metadata: name: agree-qwed-sympy-eval atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -1049,7 +1049,7 @@ metadata: name: repo-facts-reactpress-findall-column-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -1078,7 +1078,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1109,7 +1109,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1138,7 +1138,7 @@ metadata: name: agree-reactpress-findall-column-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -1165,7 +1165,7 @@ metadata: name: repo-facts-unleash-addon-webhook-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -1194,7 +1194,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1225,7 +1225,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1254,7 +1254,7 @@ metadata: name: agree-unleash-addon-webhook-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -1281,7 +1281,7 @@ metadata: name: repo-facts-winml-cli-serve-cors atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -1310,7 +1310,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1341,7 +1341,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1370,7 +1370,7 @@ metadata: name: agree-winml-cli-serve-cors atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -1397,7 +1397,7 @@ metadata: name: repo-facts-wp-directorist-avatar-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -1426,7 +1426,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1457,7 +1457,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1486,7 +1486,7 @@ metadata: name: agree-wp-directorist-avatar-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -1513,7 +1513,7 @@ metadata: name: repo-facts-wp-gopay-log-filter-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -1542,7 +1542,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1573,7 +1573,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1602,7 +1602,7 @@ metadata: name: agree-wp-gopay-log-filter-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: @@ -1629,7 +1629,7 @@ metadata: name: repo-facts-wp-groundhogg-confirm-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - repo-facts env: @@ -1658,7 +1658,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1689,7 +1689,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - verify env: @@ -1718,7 +1718,7 @@ metadata: name: agree-wp-groundhogg-confirm-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3 + image: localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4 command: - agree env: diff --git a/plane/tasks/verify.yaml b/plane/tasks/verify.yaml index c34de55..7752921 100644 --- a/plane/tasks/verify.yaml +++ b/plane/tasks/verify.yaml @@ -10,7 +10,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: "localhost:5001/ousast-runner@sha256:318cbe019c1155b43fe59e47368d99e78f6f6011b7c822292251de53cf567bf3" + image: "localhost:5001/ousast-runner@sha256:055ef8581911fbe9c3a3eb97b78b8a7206723460238aed52ce25b3d04756b3f4" command: ["verify"] resources: requests: diff --git a/plane/workspaces/budibase-mongo-template-nosqli-inputs.yaml b/plane/workspaces/budibase-mongo-template-nosqli-inputs.yaml index d3a9a3a..4a83225 100644 --- a/plane/workspaces/budibase-mongo-template-nosqli-inputs.yaml +++ b/plane/workspaces/budibase-mongo-template-nosqli-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,7 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"read\",\n \"line\": 539,\n \"path\": \"packages/server/src/integrations/mongodb.ts\"\n },\n {\n \"function\": \"enrichContext\",\n \"line\": 136,\n \"path\": \"packages/server/src/sdk/workspace/queries/queries.ts\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"budibase-mongo-template-nosqli\",\n \"sites\": [\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\",\n \"packages/server/src/integrations/mongodb.ts::read\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.003638,\n \"recorded_usd\": 0.0767\n },\n \"family\": \"injection\",\n \"id\": \"budibase-mongo-template-nosqli\",\n \"ranges\": {\n \"packages/server/src/api/routes/tests/queries/rest.spec.ts\": [\n [\n 580,\n 580\n ]\n ],\n \"packages/server/src/sdk/workspace/queries/queries.ts\": [\n [\n 2,\n 3\n ],\n [\n 15,\n 15\n ],\n [\n 120,\n 125\n ],\n [\n 136,\n 136\n ],\n [\n 138,\n 140\n ]\n ],\n \"packages/string-templates/src/index.ts\": [\n [\n 276,\n 276\n ]\n ],\n \"packages/string-templates/test/basic.spec.ts\": [\n [\n 10,\n 10\n ],\n [\n 128,\n 128\n ]\n ]\n },\n \"ranges_basis\"\ + : \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\",\n \"packages/server/src/integrations/mongodb.ts::read\"\n ],\n \"sites_in_set\": [\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\",\n \"packages/server/src/integrations/mongodb.ts::read\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"packages/server/src/integrations/mongodb.ts\",\n \"read\",\n 539\n ],\n [\n \"packages/server/src/sdk/workspace/queries/queries.ts\",\n \"enrichContext\",\n 136\n ]\n ],\n \"kept\": [\n [\n \"packages/server/src/integrations/mongodb.ts\",\n \"read\",\n 539\n ],\n [\n \"packages/server/src/sdk/workspace/queries/queries.ts\",\n \"enrichContext\",\n 136\n ]\n ],\n \"triage\": {\n \"packages/server/src/integrations/mongodb.ts::read\": \"request\",\n \"packages/server/src/sdk/workspace/queries/queries.ts::enrichContext\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/budibase-mongo-template-nosqli-vulnerable.yaml b/plane/workspaces/budibase-mongo-template-nosqli-vulnerable.yaml index ce027b8..976403b 100644 --- a/plane/workspaces/budibase-mongo-template-nosqli-vulnerable.yaml +++ b/plane/workspaces/budibase-mongo-template-nosqli-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/cct-studio-server-cmdi-inputs.yaml b/plane/workspaces/cct-studio-server-cmdi-inputs.yaml index 6d7c658..07fe45b 100644 --- a/plane/workspaces/cct-studio-server-cmdi-inputs.yaml +++ b/plane/workspaces/cct-studio-server-cmdi-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,7 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"checkAndInstallAgent\",\n \"line\": 280,\n \"path\": \"cli-tool/src/sandbox-server.js\"\n },\n {\n \"function\": \"executeLocalTask\",\n \"line\": 464,\n \"path\": \"cli-tool/src/sandbox-server.js\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"cct-studio-server-cmdi\",\n \"sites\": [\n \"cli-tool/src/sandbox-server.js::executeLocalTask\",\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.003055,\n \"recorded_usd\": 0.0242\n },\n \"family\": \"injection\",\n \"id\": \"cct-studio-server-cmdi\",\n \"ranges\": {\n \"CHANGELOG.md\": [\n [\n 7,\n 7\n ]\n ],\n \"cli-tool/package.json\": [\n [\n 3,\n 3\n ]\n ],\n \"cli-tool/src/sandbox-server.js\": [\n [\n 11,\n 11\n ],\n [\n 50,\n 50\n ],\n [\n 52,\n 52\n ],\n [\n 54,\n 54\n ],\n [\n 202,\n 202\n ],\n [\n 209,\n 209\n ],\n [\n 212,\n 214\n ],\n [\n 216,\n 217\n ],\n [\n 278,\n 278\n ],\n [\n 280,\n 280\n ],\n [\n 282,\n 283\n ],\n [\n 463,\n 464\n ],\n [\n 471,\n \ + \ 471\n ],\n [\n 561,\n 562\n ]\n ],\n \"package.json\": [\n [\n 3,\n 3\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"cli-tool/src/sandbox-server.js::executeLocalTask\",\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\"\n ],\n \"sites_in_set\": [\n \"cli-tool/src/sandbox-server.js::executeLocalTask\",\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"cli-tool/src/sandbox-server.js\",\n \"checkAndInstallAgent\",\n 280\n ],\n [\n \"cli-tool/src/sandbox-server.js\",\n \"executeLocalTask\",\n 464\n ]\n ],\n \"kept\": [\n [\n \"cli-tool/src/sandbox-server.js\",\n \"checkAndInstallAgent\",\n 280\n ],\n [\n \"cli-tool/src/sandbox-server.js\",\n \"executeLocalTask\",\n 464\n ]\n ],\n \"triage\": {\n \"cli-tool/src/sandbox-server.js::checkAndInstallAgent\": \"parameter\",\n \"cli-tool/src/sandbox-server.js::executeLocalTask\": \"request\"\n }\n}\n" diff --git a/plane/workspaces/cct-studio-server-cmdi-vulnerable.yaml b/plane/workspaces/cct-studio-server-cmdi-vulnerable.yaml index e1b08fb..30c4161 100644 --- a/plane/workspaces/cct-studio-server-cmdi-vulnerable.yaml +++ b/plane/workspaces/cct-studio-server-cmdi-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/contextforge-prompt-ssti-inputs.yaml b/plane/workspaces/contextforge-prompt-ssti-inputs.yaml index 6b84281..931cb3f 100644 --- a/plane/workspaces/contextforge-prompt-ssti-inputs.yaml +++ b/plane/workspaces/contextforge-prompt-ssti-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,16 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"_compile_jinja_template\",\n \"line\": 106,\n \"path\": \"mcpgateway/services/prompt_service.py\"\n },\n {\n \"function\": \"_render_template\",\n \"line\": 2892,\n \"path\": \"mcpgateway/services/prompt_service.py\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"contextforge-prompt-ssti\",\n \"sites\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 2,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.004212,\n \"recorded_usd\": 0.0511\n },\n \"family\": \"injection\",\n \"id\": \"contextforge-prompt-ssti\",\n \"ranges\": {\n \".env.example\": [\n [\n 164,\n 166\n ]\n ],\n \".secrets.baseline\": [\n [\n 6,\n 6\n ],\n [\n 95,\n 95\n ],\n [\n 103,\n 103\n ],\n [\n 111,\n 111\n ],\n [\n 119,\n 119\n ],\n [\n 127,\n 127\n ],\n [\n 135,\n 135\n ],\n [\n 143,\n 143\n ],\n [\n 151,\n 151\n ],\n [\n 159,\n 159\n ],\n [\n 167,\n 167\n ],\n [\n 175,\n 175\n ],\n [\n 183,\n 183\n ],\n [\n 253,\n 253\n ],\n [\n 261,\n 261\n\ + \ ],\n [\n 269,\n 269\n ],\n [\n 277,\n 277\n ],\n [\n 285,\n 285\n ],\n [\n 293,\n 293\n ],\n [\n 301,\n 301\n ],\n [\n 309,\n 309\n ],\n [\n 4215,\n 4215\n ],\n [\n 4865,\n 4865\n ],\n [\n 4873,\n 4873\n ],\n [\n 5195,\n 5195\n ],\n [\n 5957,\n 5957\n ],\n [\n 8393,\n 8393\n ],\n [\n 8401,\n 8401\n ],\n [\n 8409,\n 8409\n ],\n [\n 8759,\n 8759\n ],\n [\n 8767,\n 8767\n ],\n [\n 8775,\n 8775\n ],\n [\n 8783,\n 8783\n ],\n [\n 8791,\n 8791\n ],\n [\n 8799,\n 8799\n ],\n [\n 8807,\n 8807\n ],\n [\n 8815,\n 8815\n ],\n [\n 8823,\n 8823\n ],\n [\n 8831,\n 8831\n ],\n [\n 8839,\n 8839\n ],\n [\n 8847,\n 8847\n ],\n [\n 8855,\n 8855\n ],\n [\n 8863,\n 8863\n ],\n [\n 9237,\n 9237\n ],\n [\n 9245,\n\ + \ 9245\n ],\n [\n 9253,\n 9253\n ],\n [\n 9263,\n 9263\n ]\n ],\n \"CHANGELOG.md\": [\n [\n 4,\n 4\n ]\n ],\n \"README.md\": [\n [\n 751,\n 751\n ],\n [\n 761,\n 761\n ]\n ],\n \"SECURITY.md\": [\n [\n 276,\n 276\n ],\n [\n 374,\n 374\n ]\n ],\n \"crates/mcp_runtime/tests/runtime.rs\": [\n [\n 1647,\n 1647\n ]\n ],\n \"docs/docs/architecture/security-features.md\": [\n [\n 120,\n 120\n ]\n ],\n \"docs/docs/manage/securing.md\": [\n [\n 428,\n 428\n ],\n [\n 440,\n 440\n ]\n ],\n \"mcpgateway/admin.py\": [\n [\n 132,\n 132\n ],\n [\n 13075,\n 13075\n ],\n [\n 13196,\n 13196\n ]\n ],\n \"mcpgateway/config.py\": [\n [\n 93,\n 93\n ],\n [\n 1738,\n 1738\n ],\n [\n 1755,\n 1755\n ]\n ],\n \"mcpgateway/main.py\": [\n [\n 156,\n 156\n ],\n [\n 1820,\n 1823\n ],\n [\n 2426,\n\ + \ 2426\n ],\n [\n 6362,\n 6362\n ],\n [\n 6559,\n 6559\n ]\n ],\n \"mcpgateway/services/content_security.py\": [\n [\n 16,\n 16\n ],\n [\n 48,\n 48\n ],\n [\n 171,\n 171\n ],\n [\n 192,\n 192\n ],\n [\n 194,\n 194\n ],\n [\n 201,\n 201\n ],\n [\n 204,\n 204\n ],\n [\n 333,\n 333\n ],\n [\n 335,\n 335\n ],\n [\n 345,\n 347\n ],\n [\n 351,\n 352\n ],\n [\n 357,\n 357\n ],\n [\n 360,\n 360\n ],\n [\n 362,\n 363\n ]\n ],\n \"mcpgateway/services/metrics.py\": [\n [\n 111,\n 111\n ],\n [\n 121,\n 127\n ],\n [\n 130,\n 131\n ]\n ],\n \"mcpgateway/services/prompt_service.py\": [\n [\n 27,\n 27\n ],\n [\n 53,\n 53\n ],\n [\n 77,\n 77\n ],\n [\n 80,\n 80\n ],\n [\n 82,\n 82\n ],\n [\n 84,\n 84\n ],\n [\n 88,\n 88\n ],\n [\n\ + \ 730,\n 730\n ],\n [\n 767,\n 768\n ],\n [\n 951,\n 951\n ],\n [\n 1014,\n 1014\n ],\n [\n 1101,\n 1102\n ],\n [\n 1228,\n 1228\n ],\n [\n 1275,\n 1275\n ],\n [\n 2169,\n 2169\n ],\n [\n 2276,\n 2276\n ],\n [\n 2278,\n 2278\n ],\n [\n 2446,\n 2446\n ],\n [\n 2889,\n 2889\n ]\n ],\n \"mcpgateway/services/resource_service.py\": [\n [\n 67,\n 67\n ],\n [\n 460,\n 463\n ],\n [\n 529,\n 533\n ],\n [\n 535,\n 535\n ],\n [\n 537,\n 537\n ],\n [\n 685,\n 685\n ],\n [\n 700,\n 700\n ],\n [\n 714,\n 714\n ],\n [\n 838,\n 844\n ],\n [\n 847,\n 847\n ],\n [\n 852,\n 852\n ],\n [\n 872,\n 872\n ],\n [\n 893,\n 893\n ],\n [\n 922,\n 922\n ],\n [\n 2893,\n 2896\n ],\n [\n 2983,\n 2984\n ],\n [\n 2987,\n 3017\n \ + \ ],\n [\n 3018,\n 3018\n ],\n [\n 3029,\n 3029\n ],\n [\n 3037,\n 3037\n ],\n [\n 3204,\n 3204\n ]\n ],\n \"mcpgateway/services/tool_service.py\": [\n [\n 64,\n 74\n ],\n [\n 79,\n 79\n ],\n [\n 949,\n 949\n ],\n [\n 1873,\n 1873\n ],\n [\n 2310,\n 2310\n ],\n [\n 5961,\n 5961\n ]\n ],\n \"tests/e2e/test_admin_apis.py\": [\n [\n 29,\n 29\n ],\n [\n 36,\n 38\n ],\n [\n 116,\n 116\n ],\n [\n 125,\n 127\n ],\n [\n 149,\n 149\n ],\n [\n 151,\n 151\n ],\n [\n 690,\n 690\n ]\n ],\n \"tests/e2e/test_main_apis.py\": [\n [\n 48,\n 48\n ],\n [\n 55,\n 55\n ],\n [\n 58,\n 58\n ],\n [\n 63,\n 63\n ],\n [\n 980,\n 980\n ],\n [\n 1021,\n 1021\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ],\n \"tests/unit/mcpgateway/services/test_content_security.py\"\ + : [\n [\n 5,\n 7\n ],\n [\n 13,\n 13\n ],\n [\n 15,\n 22\n ],\n [\n 49,\n 49\n ],\n [\n 209,\n 209\n ],\n [\n 306,\n 306\n ],\n [\n 323,\n 323\n ],\n [\n 334,\n 336\n ],\n [\n 338,\n 339\n ],\n [\n 342,\n 342\n ],\n [\n 344,\n 345\n ],\n [\n 348,\n 348\n ],\n [\n 363,\n 363\n ],\n [\n 365,\n 365\n ],\n [\n 368,\n 368\n ],\n [\n 373,\n 373\n ],\n [\n 389,\n 389\n ],\n [\n 401,\n 402\n ],\n [\n 404,\n 405\n ],\n [\n 408,\n 408\n ],\n [\n 410,\n 411\n ],\n [\n 414,\n 414\n ],\n [\n 422,\n 422\n ],\n [\n 434,\n 473\n ],\n [\n 479,\n 479\n ],\n [\n 504,\n 504\n ],\n [\n 516,\n 516\n ],\n [\n 526,\n 526\n ],\n [\n 537,\n 537\n ],\n [\n 548,\n 548\n ],\n [\n 582,\n 582\n ],\n [\n \ + \ 614,\n 614\n ]\n ],\n \"tests/unit/mcpgateway/services/test_prompt_service.py\": [\n [\n 33,\n 39\n ],\n [\n 378,\n 378\n ],\n [\n 382,\n 385\n ],\n [\n 387,\n 387\n ],\n [\n 1178,\n 1178\n ],\n [\n 1963,\n 1963\n ],\n [\n 1967,\n 1967\n ],\n [\n 1972,\n 1972\n ],\n [\n 1984,\n 1989\n ],\n [\n 1991,\n 1992\n ]\n ],\n \"tests/unit/mcpgateway/services/test_resource_service.py\": [\n [\n 31,\n 35\n ],\n [\n 2309,\n 2333\n ],\n [\n 2413,\n 2414\n ],\n [\n 2551,\n 2552\n ],\n [\n 2555,\n 2556\n ],\n [\n 2560,\n 2560\n ],\n [\n 2565,\n 2566\n ],\n [\n 2570,\n 2570\n ],\n [\n 2572,\n 2574\n ],\n [\n 2577,\n 2578\n ],\n [\n 2587,\n 2588\n ],\n [\n 2593,\n 2593\n ],\n [\n 2595,\n 2595\n ],\n [\n 2598,\n 2599\n ],\n [\n 2603,\n 2606\n \ + \ ],\n [\n 2608,\n 2608\n ],\n [\n 2610,\n 2612\n ],\n [\n 2614,\n 2615\n ],\n [\n 2618,\n 2619\n ],\n [\n 2622,\n 2622\n ],\n [\n 2637,\n 2638\n ],\n [\n 2642,\n 2642\n ],\n [\n 2644,\n 2645\n ],\n [\n 2724,\n 2753\n ]\n ],\n \"tests/unit/mcpgateway/services/test_row_level_locking.py\": [\n [\n 14,\n 14\n ],\n [\n 16,\n 16\n ],\n [\n 19,\n 24\n ],\n [\n 26,\n 26\n ],\n [\n 77,\n 77\n ],\n [\n 82,\n 82\n ],\n [\n 143,\n 143\n ],\n [\n 442,\n 442\n ],\n [\n 509,\n 509\n ],\n [\n 510,\n 510\n ],\n [\n 515,\n 515\n ],\n [\n 516,\n 516\n ],\n [\n 521,\n 521\n ]\n ],\n \"tests/unit/mcpgateway/test_admin.py\": [\n [\n 281,\n 285\n ],\n [\n 2523,\n 2523\n ],\n [\n 2590,\n 2628\n ],\n [\n 2935,\n 2970\n ]\n ],\n \"tests/unit/mcpgateway/test_main.py\"\ + : [\n [\n 37,\n 49\n ],\n [\n 487,\n 487\n ],\n [\n 490,\n 490\n ],\n [\n 521,\n 521\n ],\n [\n 524,\n 524\n ],\n [\n 764,\n 764\n ],\n [\n 787,\n 787\n ],\n [\n 1198,\n 1198\n ],\n [\n 1208,\n 1208\n ],\n [\n 1211,\n 1211\n ],\n [\n 1414,\n 1414\n ],\n [\n 1416,\n 1416\n ],\n [\n 1420,\n 1420\n ],\n [\n 1432,\n 1432\n ],\n [\n 1441,\n 1441\n ],\n [\n 2407,\n 2407\n ],\n [\n 2476,\n 2476\n ]\n ],\n \"tests/unit/mcpgateway/test_main_error_handlers.py\": [\n [\n 12,\n 13\n ],\n [\n 18,\n 20\n ],\n [\n 21,\n 21\n ],\n [\n 22,\n 22\n ],\n [\n 26,\n 33\n ],\n [\n 44,\n 45\n ],\n [\n 577,\n 577\n ],\n [\n 600,\n 600\n ],\n [\n 607,\n 607\n ],\n [\n 612,\n 612\n ],\n [\n 618,\n 618\n ]\n ]\n },\n \"ranges_basis\"\ + : \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ],\n \"sites_in_set\": [\n \"mcpgateway/services/prompt_service.py::_render_template\",\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"kept\": [\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_compile_jinja_template\",\n 106\n ],\n [\n \"mcpgateway/services/prompt_service.py\",\n \"_render_template\",\n 2892\n ]\n ],\n \"triage\": {\n \"mcpgateway/services/prompt_service.py::_compile_jinja_template\": \"stored\",\n \"mcpgateway/services/prompt_service.py::_render_template\": \"stored\"\n }\n}\n" diff --git a/plane/workspaces/contextforge-prompt-ssti-vulnerable.yaml b/plane/workspaces/contextforge-prompt-ssti-vulnerable.yaml index 47236ba..f711185 100644 --- a/plane/workspaces/contextforge-prompt-ssti-vulnerable.yaml +++ b/plane/workspaces/contextforge-prompt-ssti-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/froxlor-admin-ip-second-order-sqli-inputs.yaml b/plane/workspaces/froxlor-admin-ip-second-order-sqli-inputs.yaml index 4bac379..c462f67 100644 --- a/plane/workspaces/froxlor-admin-ip-second-order-sqli-inputs.yaml +++ b/plane/workspaces/froxlor-admin-ip-second-order-sqli-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,7 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"validateIpAddresses\",\n \"line\": 1050,\n \"path\": \"lib/Froxlor/Api/Commands/Domains.php\"\n },\n {\n \"function\": \"listing\",\n \"line\": 76,\n \"path\": \"lib/Froxlor/Api/Commands/IpsAndPorts.php\"\n },\n {\n \"function\": \"listingCount\",\n \"line\": 108,\n \"path\": \"lib/Froxlor/Api/Commands/IpsAndPorts.php\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"froxlor-admin-ip-second-order-sqli\",\n \"sites\": [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\",\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 3,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.007758,\n \"recorded_usd\": 0.0873\n },\n \"family\": \"injection\",\n \"id\": \"froxlor-admin-ip-second-order-sqli\",\n \"ranges\": {\n \"lib/Froxlor/Api/Commands/Admins.php\": [\n [\n 273,\n 273\n ],\n [\n 358,\n 358\n ],\n [\n 619,\n 619\n ],\n [\n 729,\n 729\n ]\n ],\n \"lib/Froxlor/Api/Commands/Domains.php\": [\n [\n 1016,\n 1016\n ]\n ],\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\": [\n [\n 72,\n 72\n ],\n [\n 104,\n 104\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\"\ + ,\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\",\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\"\n ],\n \"sites_in_set\": [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\",\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"lib/Froxlor/Api/Commands/Domains.php\",\n \"validateIpAddresses\",\n 1050\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listing\",\n 76\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listingCount\",\n 108\n ]\n ],\n \"kept\": [\n [\n \"lib/Froxlor/Api/Commands/Domains.php\",\n \"validateIpAddresses\",\n 1050\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listing\",\n 76\n ],\n [\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php\",\n \"listingCount\",\n 108\n ]\n ],\n \"triage\": {\n \"lib/Froxlor/Api/Commands/Domains.php::validateIpAddresses\": \"parameter\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listing\": \"request\",\n \"lib/Froxlor/Api/Commands/IpsAndPorts.php::listingCount\": \"request\"\n }\n}\n" diff --git a/plane/workspaces/froxlor-admin-ip-second-order-sqli-vulnerable.yaml b/plane/workspaces/froxlor-admin-ip-second-order-sqli-vulnerable.yaml index c1eeccc..2b4d437 100644 --- a/plane/workspaces/froxlor-admin-ip-second-order-sqli-vulnerable.yaml +++ b/plane/workspaces/froxlor-admin-ip-second-order-sqli-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/fuxa-tdengine-escape-sqli-inputs.yaml b/plane/workspaces/fuxa-tdengine-escape-sqli-inputs.yaml index 16bad38..95a850b 100644 --- a/plane/workspaces/fuxa-tdengine-escape-sqli-inputs.yaml +++ b/plane/workspaces/fuxa-tdengine-escape-sqli-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,6 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"getDaqValue\",\n \"line\": 89,\n \"path\": \"server/runtime/storage/tdengine/index.js\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"fuxa-tdengine-escape-sqli\",\n \"sites\": [\n \"server/runtime/storage/tdengine/index.js::getDaqValue\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.000833,\n \"recorded_usd\": 0.1527\n },\n \"family\": \"injection\",\n \"id\": \"fuxa-tdengine-escape-sqli\",\n \"ranges\": {\n \"server/runtime/storage/tdengine/index.js\": [\n [\n 11,\n 11\n ],\n [\n 121,\n 121\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"server/runtime/storage/tdengine/index.js::getDaqValue\"\n ],\n \"sites_in_set\": [\n \"server/runtime/storage/tdengine/index.js::getDaqValue\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"server/runtime/storage/tdengine/index.js\",\n \"getDaqValue\",\n 89\n ]\n ],\n \"kept\": [\n [\n \"server/runtime/storage/tdengine/index.js\",\n \"getDaqValue\",\n 89\n ]\n ],\n \"triage\": {\n \"server/runtime/storage/tdengine/index.js::getDaqValue\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/fuxa-tdengine-escape-sqli-vulnerable.yaml b/plane/workspaces/fuxa-tdengine-escape-sqli-vulnerable.yaml index 282e2cb..f5a4e8e 100644 --- a/plane/workspaces/fuxa-tdengine-escape-sqli-vulnerable.yaml +++ b/plane/workspaces/fuxa-tdengine-escape-sqli-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/lmdeploy-media-url-ssrf-inputs.yaml b/plane/workspaces/lmdeploy-media-url-ssrf-inputs.yaml index 5cfa187..bd09f7f 100644 --- a/plane/workspaces/lmdeploy-media-url-ssrf-inputs.yaml +++ b/plane/workspaces/lmdeploy-media-url-ssrf-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,9 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"_load_http_url\",\n \"line\": 68,\n \"path\": \"lmdeploy/vl/media/connection.py\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"lmdeploy-media-url-ssrf\",\n \"sites\": [\n \"lmdeploy/vl/media/connection.py::_load_http_url\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.000768,\n \"recorded_usd\": 0.0249\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"lmdeploy-media-url-ssrf\",\n \"ranges\": {\n \"docs/en/multi_modal/multimodal_inputs.md\": [\n [\n 395,\n 395\n ]\n ],\n \"docs/zh_cn/multi_modal/multimodal_inputs.md\": [\n [\n 394,\n 394\n ]\n ],\n \"lmdeploy/api.py\": [\n [\n 21,\n 21\n ],\n [\n 45,\n 45\n ],\n [\n 78,\n 78\n ]\n ],\n \"lmdeploy/cli/serve.py\": [\n [\n 77,\n 77\n ],\n [\n 325,\n 325\n ],\n [\n 358,\n 358\n ]\n ],\n \"lmdeploy/pipeline.py\": [\n [\n 42,\n 42\n ],\n [\n 53,\n 53\n ],\n [\n 84,\n 84\n\ + \ ],\n [\n 89,\n 89\n ],\n [\n 115,\n 115\n ],\n [\n 165,\n 165\n ],\n [\n 204,\n 204\n ]\n ],\n \"lmdeploy/serve/core/async_engine.py\": [\n [\n 116,\n 116\n ],\n [\n 125,\n 125\n ]\n ],\n \"lmdeploy/serve/core/vl_async_engine.py\": [\n [\n 20,\n 20\n ],\n [\n 46,\n 46\n ],\n [\n 52,\n 52\n ]\n ],\n \"lmdeploy/serve/openai/api_server.py\": [\n [\n 1528,\n 1528\n ],\n [\n 1580,\n 1580\n ],\n [\n 1620,\n 1620\n ]\n ],\n \"lmdeploy/serve/processors/multimodal.py\": [\n [\n 40,\n 40\n ],\n [\n 47,\n 47\n ],\n [\n 52,\n 52\n ],\n [\n 106,\n 107\n ],\n [\n 156,\n 156\n ],\n [\n 163,\n 163\n ],\n [\n 167,\n 167\n ],\n [\n 170,\n 170\n ],\n [\n 180,\n 180\n ],\n [\n 192,\n 192\n ],\n [\n 269,\n 269\n ],\n [\n 282,\n 282\n ],\n [\n 313,\n\ + \ 313\n ],\n [\n 315,\n 316\n ],\n [\n 329,\n 329\n ],\n [\n 403,\n 403\n ]\n ],\n \"lmdeploy/vl/media/connection.py\": [\n [\n 7,\n 7\n ],\n [\n 10,\n 10\n ],\n [\n 28,\n 28\n ],\n [\n 34,\n 34\n ],\n [\n 54,\n 54\n ],\n [\n 56,\n 59\n ],\n [\n 67,\n 69\n ],\n [\n 71,\n 71\n ],\n [\n 95,\n 95\n ],\n [\n 100,\n 100\n ]\n ],\n \"tests/test_lmdeploy/test_content_merge.py\": [\n [\n 12,\n 12\n ],\n [\n 245,\n 245\n ],\n [\n 307,\n 307\n ],\n [\n 335,\n 335\n ],\n [\n 340,\n 340\n ],\n [\n 385,\n 385\n ],\n [\n 407,\n 407\n ],\n [\n 415,\n 415\n ]\n ],\n \"tests/test_lmdeploy/test_vl/test_safe_url.py\": [\n [\n 43,\n 44\n ],\n [\n 47,\n 47\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case,\ + \ 'old')\",\n \"sites\": [\n \"lmdeploy/vl/media/connection.py::_load_http_url\"\n ],\n \"sites_in_set\": [\n \"lmdeploy/vl/media/connection.py::_load_http_url\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"lmdeploy/vl/media/connection.py\",\n \"_load_http_url\",\n 68\n ]\n ],\n \"kept\": [\n [\n \"lmdeploy/vl/media/connection.py\",\n \"_load_http_url\",\n 68\n ]\n ],\n \"triage\": {\n \"lmdeploy/vl/media/connection.py::_load_http_url\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/lmdeploy-media-url-ssrf-vulnerable.yaml b/plane/workspaces/lmdeploy-media-url-ssrf-vulnerable.yaml index b6a63c1..e4e34dd 100644 --- a/plane/workspaces/lmdeploy-media-url-ssrf-vulnerable.yaml +++ b/plane/workspaces/lmdeploy-media-url-ssrf-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/medplum-external-auth-open-redirect-inputs.yaml b/plane/workspaces/medplum-external-auth-open-redirect-inputs.yaml index 45872e7..673a012 100644 --- a/plane/workspaces/medplum-external-auth-open-redirect-inputs.yaml +++ b/plane/workspaces/medplum-external-auth-open-redirect-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,8 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"externalCallbackHandler\",\n \"line\": 166,\n \"path\": \"packages/server/src/auth/external.ts\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"medplum-external-auth-open-redirect\",\n \"sites\": [\n \"packages/server/src/auth/external.ts::externalCallbackHandler\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.001509,\n \"recorded_usd\": 0.0228\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"medplum-external-auth-open-redirect\",\n \"ranges\": {\n \"packages/definitions/src/fhir/r4/profiles-medplum.json\": [\n [\n 890,\n 904\n ],\n [\n 1080,\n 1080\n ]\n ],\n \"packages/docs/static/data/medplumDefinitions/clientapplication.json\": [\n [\n 308,\n 308\n ],\n [\n 315,\n 315\n ],\n [\n 317,\n 317\n ],\n [\n 319,\n 319\n ],\n [\n 324,\n 324\n ],\n [\n 331,\n 331\n ],\n [\n 333,\n 333\n ],\n [\n 335,\n 335\n ],\n [\n 340,\n 340\n ],\n [\n 344,\n 344\n ],\n\ + \ [\n 347,\n 347\n ],\n [\n 348,\n 348\n ],\n [\n 351,\n 351\n ],\n [\n 450,\n 450\n ]\n ],\n \"packages/docs/static/data/medplumDefinitions/smartapplaunch.json\": [\n [\n 206,\n 206\n ]\n ],\n \"packages/fhirtypes/dist/ClientApplication.d.ts\": [\n [\n 137,\n 141\n ],\n [\n 154,\n 154\n ],\n [\n 191,\n 191\n ],\n [\n 195,\n 195\n ],\n [\n 197,\n 197\n ],\n [\n 200,\n 200\n ],\n [\n 202,\n 202\n ],\n [\n 205,\n 205\n ],\n [\n 207,\n 207\n ],\n [\n 211,\n 211\n ],\n [\n 213,\n 213\n ],\n [\n 216,\n 216\n ],\n [\n 218,\n 218\n ],\n [\n 221,\n 221\n ],\n [\n 223,\n 223\n ]\n ],\n \"packages/server/src/auth/external.test.ts\": [\n [\n 410,\n 410\n ]\n ],\n \"packages/server/src/auth/external.ts\": [\n [\n 13,\n 13\n ],\n [\n 18,\n 18\n ],\n [\n 124,\n\ + \ 133\n ],\n [\n 139,\n 139\n ],\n [\n 283,\n 283\n ]\n ],\n \"packages/server/src/oauth/clients.ts\": [\n [\n 69,\n 69\n ],\n [\n 76,\n 76\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"packages/server/src/auth/external.ts::externalCallbackHandler\"\n ],\n \"sites_in_set\": [\n \"packages/server/src/auth/external.ts::externalCallbackHandler\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"packages/server/src/auth/external.ts\",\n \"externalCallbackHandler\",\n 166\n ]\n ],\n \"kept\": [\n [\n \"packages/server/src/auth/external.ts\",\n \"externalCallbackHandler\",\n 166\n ]\n ],\n \"triage\": {\n \"packages/server/src/auth/external.ts::externalCallbackHandler\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/medplum-external-auth-open-redirect-vulnerable.yaml b/plane/workspaces/medplum-external-auth-open-redirect-vulnerable.yaml index 2ecc1a2..ae5a084 100644 --- a/plane/workspaces/medplum-external-auth-open-redirect-vulnerable.yaml +++ b/plane/workspaces/medplum-external-auth-open-redirect-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/pgadmin-maintenance-sqli-inputs.yaml b/plane/workspaces/pgadmin-maintenance-sqli-inputs.yaml index b59693d..6d642f6 100644 --- a/plane/workspaces/pgadmin-maintenance-sqli-inputs.yaml +++ b/plane/workspaces/pgadmin-maintenance-sqli-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -17,7 +17,8 @@ spec: \n },\n {\n \"function\": \"run_job\",\n \"line\": 746,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\"\n },\n {\n \"function\": \"create\",\n \"line\": 365,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_schedules/__init__.py\"\n },\n {\n \"function\": \"msql\",\n \"line\": 454,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_schedules/__init__.py\"\n },\n {\n \"function\": \"sql\",\n \"line\": 492,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_schedules/__init__.py\"\n },\n {\n \"function\": \"msql\",\n \"line\": 623,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/event_triggers/__init__.py\"\n },\n {\n \"function\": \"create\",\n \"line\": 290,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/extensions/__init__.py\"\n },\n {\n \"function\"\ : \"delete\",\n \"line\": 394,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/extensions/__init__.py\"\n },\n {\n \"function\": \"get_extension_details\",\n \"line\": 30,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/extensions/utils.py\"\n },\n {\n \"function\": \"delete\",\n \"line\": 601,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/foreign_data_wrappers/__init__.py\"\n },\n {\n \"function\": \"create\",\n \"line\": 464,\n \"path\": \"web/pgadmin/browser/server_groups/servers/databases/foreign_data_wrappers/foreign_servers/__init__.py\"\n },\n {\n \"function\": \"create_maintenance_job\",\n \"line\": 222,\n \"path\": \"web/pgadmin/tools/maintenance/__init__.py\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"pgadmin-maintenance-sqli\",\n \"sites\": [\n \"web/pgadmin/tools/maintenance/__init__.py::create_maintenance_job\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 27,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.048665,\n \"recorded_usd\": 0.3949\n },\n \"family\": \"injection\",\n \"id\": \"pgadmin-maintenance-sqli\",\n \"ranges\": {\n \"web/pgadmin/tools/maintenance/__init__.py\": [\n [\n 12,\n 12\n ],\n [\n 150,\n 150\n ],\n [\n 171,\n 171\n ]\n ],\n \"web/pgadmin/tools/maintenance/templates/maintenance/sql/command.sql\": [\n [\n 16,\n 16\n ]\n ],\n \"web/pgadmin/tools/maintenance/tests/test_maintenance_create_job_unit_test.py\": [\n [\n 562,\n 562\n ],\n [\n 647,\n 647\n ],\n [\n 714,\n 714\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case,\ + \ 'old')\",\n \"sites\": [\n \"web/pgadmin/tools/maintenance/__init__.py::create_maintenance_job\"\n ],\n \"sites_in_set\": [\n \"web/pgadmin/tools/maintenance/__init__.py::create_maintenance_job\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"web/pgadmin/authenticate/ldap.py\",\n \"login\",\n 169\n ],\n [\n \"web/pgadmin/authenticate/ldap.py\",\n \"search_ldap_user\",\n 288\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/__init__.py\",\n \"create_restore_point\",\n 1881\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"_check_rename_db_or_change_table_space\",\n 808\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"_fetch_db_details\",\n 831\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"create\",\n 761\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"delete\",\n 1061\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"get_ctypes\",\n 651\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"get_encodings\",\n\ \ 623\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"get_sql\",\n 1107\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/__init__.py\",\n \"update\",\n 936\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/casts/__init__.py\",\n \"delete\",\n 546\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"_fetch_properties\",\n 362\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"create\",\n 412\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"enable_disable\",\n 662\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\",\n \"get_job_args_value\",\n 779\n ],\n [\n \"web/pgadmin/browser/server_groups/servers/databases/dbms_job_scheduler/dbms_jobs/__init__.py\"\ diff --git a/plane/workspaces/pgadmin-maintenance-sqli-vulnerable.yaml b/plane/workspaces/pgadmin-maintenance-sqli-vulnerable.yaml index 018ab93..67ae889 100644 --- a/plane/workspaces/pgadmin-maintenance-sqli-vulnerable.yaml +++ b/plane/workspaces/pgadmin-maintenance-sqli-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/qwed-sympy-eval-inputs.yaml b/plane/workspaces/qwed-sympy-eval-inputs.yaml index 9c6819e..4aa0fa7 100644 --- a/plane/workspaces/qwed-sympy-eval-inputs.yaml +++ b/plane/workspaces/qwed-sympy-eval-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,7 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"_verify_item\",\n \"line\": 239,\n \"path\": \"src/qwed_new/core/batch.py\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"qwed-sympy-eval\",\n \"sites\": [\n \"src/qwed_new/api/main.py::verify_math\",\n \"src/qwed_new/core/batch.py::_verify_item\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.00149,\n \"recorded_usd\": 0.022\n },\n \"family\": \"injection\",\n \"id\": \"qwed-sympy-eval\",\n \"ranges\": {\n \"src/qwed_new/api/main.py\": [\n [\n 460,\n 460\n ],\n [\n 475,\n 476\n ],\n [\n 504,\n 504\n ]\n ],\n \"src/qwed_new/core/batch.py\": [\n [\n 223,\n 223\n ],\n [\n 229,\n 230\n ],\n [\n 239,\n 239\n ],\n [\n 0,\n 0\n ]\n ],\n \"src/qwed_new/core/validator.py\": [\n [\n 13,\n 13\n ],\n [\n 64,\n 64\n ]\n ],\n \"src/qwed_new/core/verifier.py\": [\n [\n 21,\n 21\n ],\n [\n 123,\n 123\n ],\n [\n 200,\n 201\n ],\n [\n 284,\n 286\n\ + \ ],\n [\n 331,\n 333\n ],\n [\n 337,\n 338\n ],\n [\n 388,\n 390\n ],\n [\n 398,\n 398\n ],\n [\n 0,\n 0\n ]\n ],\n \"tests/test_api_exceptions.py\": [\n [\n 34,\n 34\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"src/qwed_new/api/main.py::verify_math\",\n \"src/qwed_new/core/batch.py::_verify_item\"\n ],\n \"sites_in_set\": [\n \"src/qwed_new/core/batch.py::_verify_item\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"src/qwed_new/core/batch.py\",\n \"_verify_item\",\n 239\n ]\n ],\n \"kept\": [\n [\n \"src/qwed_new/core/batch.py\",\n \"_verify_item\",\n 239\n ]\n ],\n \"triage\": {\n \"src/qwed_new/core/batch.py::_verify_item\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/qwed-sympy-eval-vulnerable.yaml b/plane/workspaces/qwed-sympy-eval-vulnerable.yaml index 4e29528..e6eccef 100644 --- a/plane/workspaces/qwed-sympy-eval-vulnerable.yaml +++ b/plane/workspaces/qwed-sympy-eval-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/reactpress-findall-column-sqli-inputs.yaml b/plane/workspaces/reactpress-findall-column-sqli-inputs.yaml index 806ffae..3db4f88 100644 --- a/plane/workspaces/reactpress-findall-column-sqli-inputs.yaml +++ b/plane/workspaces/reactpress-findall-column-sqli-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,8 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"findAll\",\n \"line\": 108,\n \"path\": \"server/src/modules/article/article.service.ts\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"reactpress-findall-column-sqli\",\n \"sites\": [\n \"server/src/modules/article/article.service.ts::findAll\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.002177,\n \"recorded_usd\": 0.0193\n },\n \"family\": \"injection\",\n \"id\": \"reactpress-findall-column-sqli\",\n \"ranges\": {\n \"server/src/modules/article/article.service.ts\": [\n [\n 6,\n 6\n ],\n [\n 106,\n 110\n ],\n [\n 307,\n 308\n ]\n ],\n \"server/src/modules/comment/comment.controller.ts\": [\n [\n 20,\n 20\n ]\n ],\n \"server/src/modules/comment/comment.service.ts\": [\n [\n 5,\n 5\n ],\n [\n 6,\n 6\n ],\n [\n 45,\n 45\n ],\n [\n 103,\n 107\n ]\n ],\n \"server/src/modules/file/file.service.ts\": [\n [\n 7,\n 7\n ],\n [\n 49,\n 49\n ],\n [\n 75,\n\ + \ 79\n ]\n ],\n \"server/src/modules/knowledge/knowledge.service.ts\": [\n [\n 6,\n 6\n ],\n [\n 137,\n 141\n ]\n ],\n \"server/src/modules/page/page.service.ts\": [\n [\n 6,\n 6\n ],\n [\n 49,\n 53\n ]\n ],\n \"server/src/starter.ts\": [\n [\n 22,\n 22\n ],\n [\n 34,\n 34\n ],\n [\n 39,\n 39\n ],\n [\n 43,\n 43\n ],\n [\n 49,\n 49\n ],\n [\n 51,\n 51\n ],\n [\n 54,\n 54\n ],\n [\n 71,\n 71\n ],\n [\n 76,\n 76\n ],\n [\n 80,\n 80\n ],\n [\n 82,\n 82\n ],\n [\n 85,\n 85\n ],\n [\n 90,\n 90\n ],\n [\n 103,\n 103\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"server/src/modules/article/article.service.ts::findAll\"\n ],\n \"sites_in_set\"\ + : [\n \"server/src/modules/article/article.service.ts::findAll\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"server/src/modules/article/article.service.ts\",\n \"findAll\",\n 108\n ]\n ],\n \"kept\": [\n [\n \"server/src/modules/article/article.service.ts\",\n \"findAll\",\n 108\n ]\n ],\n \"triage\": {\n \"server/src/modules/article/article.service.ts::findAll\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/reactpress-findall-column-sqli-vulnerable.yaml b/plane/workspaces/reactpress-findall-column-sqli-vulnerable.yaml index 99c3f11..39cc1f3 100644 --- a/plane/workspaces/reactpress-findall-column-sqli-vulnerable.yaml +++ b/plane/workspaces/reactpress-findall-column-sqli-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/unleash-addon-webhook-ssrf-inputs.yaml b/plane/workspaces/unleash-addon-webhook-ssrf-inputs.yaml index b50d39a..20a3ba5 100644 --- a/plane/workspaces/unleash-addon-webhook-ssrf-inputs.yaml +++ b/plane/workspaces/unleash-addon-webhook-ssrf-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,9 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"fetchRetry\",\n \"line\": 64,\n \"path\": \"src/lib/addons/addon.ts\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"unleash-addon-webhook-ssrf\",\n \"sites\": [\n \"src/lib/addons/addon.ts::fetchRetry\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.000514,\n \"recorded_usd\": 0.0293\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"unleash-addon-webhook-ssrf\",\n \"ranges\": {\n \"src/lib/__snapshots__/create-config.test.ts.snap\": [\n [\n 16,\n 16\n ]\n ],\n \"src/lib/addons/addon.ts\": [\n [\n 1,\n 1\n ],\n [\n 10,\n 10\n ],\n [\n 24,\n 24\n ],\n [\n 31,\n 31\n ],\n [\n 34,\n 34\n ],\n [\n 60,\n 60\n ],\n [\n 64,\n 65\n ],\n [\n 66,\n 66\n ],\n [\n 68,\n 68\n ],\n [\n 70,\n 70\n ],\n [\n 72,\n 72\n ],\n [\n 74,\n 75\n ],\n [\n 77,\n 77\n ],\n [\n 98,\n 98\n\ + \ ]\n ],\n \"src/lib/addons/datadog-definition.ts\": [\n [\n 31,\n 31\n ]\n ],\n \"src/lib/addons/datadog.test.ts\": [\n [\n 29,\n 29\n ]\n ],\n \"src/lib/addons/new-relic.test.ts\": [\n [\n 192,\n 192\n ]\n ],\n \"src/lib/addons/new-relic.ts\": [\n [\n 127,\n 127\n ]\n ],\n \"src/lib/addons/slack.test.ts\": [\n [\n 303,\n 304\n ]\n ],\n \"src/lib/addons/teams-workflow.test.ts\": [\n [\n 173,\n 173\n ]\n ],\n \"src/lib/addons/teams-workflow.ts\": [\n [\n 137,\n 137\n ]\n ],\n \"src/lib/addons/teams.test.ts\": [\n [\n 253,\n 253\n ]\n ],\n \"src/lib/addons/teams.ts\": [\n [\n 119,\n 119\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ],\n \"src/lib/create-config.ts\": [\n [\n 589,\n 589\n ],\n [\n 862,\n 862\n ]\n ],\n \"src/lib/features/metrics/integrations/integration-metrics.ts\": [\n [\n 15,\n 15\n ],\n [\n 44,\n 44\n\ + \ ]\n ],\n \"src/lib/services/addon-service.test.ts\": [\n [\n 55,\n 55\n ],\n [\n 64,\n 64\n ],\n [\n 160,\n 160\n ],\n [\n 194,\n 194\n ],\n [\n 242,\n 242\n ],\n [\n 305,\n 305\n ],\n [\n 354,\n 354\n ],\n [\n 392,\n 392\n ],\n [\n 429,\n 429\n ],\n [\n 494,\n 494\n ]\n ],\n \"src/lib/services/addon-service.ts\": [\n [\n 33,\n 33\n ],\n [\n 60,\n 60\n ],\n [\n 70,\n 70\n ],\n [\n 73,\n 73\n ],\n [\n 85,\n 85\n ],\n [\n 94,\n 94\n ],\n [\n 239,\n 239\n ],\n [\n 273,\n 273\n ],\n [\n 362,\n 362\n ]\n ],\n \"src/lib/types/model.ts\": [\n [\n 474,\n 474\n ]\n ],\n \"src/lib/types/option.ts\": [\n [\n 189,\n 189\n ],\n [\n 320,\n 320\n ]\n ],\n \"src/test/e2e/api/admin/addon.e2e.test.ts\": [\n [\n 19,\n 19\n ]\n ],\n \"src/test/e2e/services/addon-service.e2e.test.ts\"\ + : [\n [\n 25,\n 25\n ],\n [\n 41,\n 41\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"src/lib/addons/addon.ts::fetchRetry\"\n ],\n \"sites_in_set\": [\n \"src/lib/addons/addon.ts::fetchRetry\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"src/lib/addons/addon.ts\",\n \"fetchRetry\",\n 64\n ]\n ],\n \"kept\": [\n [\n \"src/lib/addons/addon.ts\",\n \"fetchRetry\",\n 64\n ]\n ],\n \"triage\": {\n \"src/lib/addons/addon.ts::fetchRetry\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/unleash-addon-webhook-ssrf-vulnerable.yaml b/plane/workspaces/unleash-addon-webhook-ssrf-vulnerable.yaml index 9ac1a53..74fa165 100644 --- a/plane/workspaces/unleash-addon-webhook-ssrf-vulnerable.yaml +++ b/plane/workspaces/unleash-addon-webhook-ssrf-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/winml-cli-serve-cors-inputs.yaml b/plane/workspaces/winml-cli-serve-cors-inputs.yaml index 5149ec9..ac0ddcc 100644 --- a/plane/workspaces/winml-cli-serve-cors-inputs.yaml +++ b/plane/workspaces/winml-cli-serve-cors-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -13,6 +13,8 @@ spec: ' - path: case.json - content: "{\n \"family\": \"config_secrets\",\n \"id\": \"winml-cli-serve-cors\",\n \"sites\": [\n \"src/winml/modelkit/serve/cli_api.py::\",\n \"src/winml/modelkit/serve/app.py::create_app\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.0021,\n \"recorded_usd\": 0.0021\n },\n \"family\": \"config_secrets\",\n \"id\": \"winml-cli-serve-cors\",\n \"ranges\": {\n \".github/workflows/modelkit-ci.yml\": [\n [\n 38,\n 38\n ],\n [\n 53,\n 53\n ]\n ],\n \"src/winml/modelkit/commands/serve.py\": [\n [\n 137,\n 137\n ],\n [\n 174,\n 174\n ]\n ],\n \"src/winml/modelkit/loader/_autoconfig.py\": [\n [\n 156,\n 156\n ]\n ],\n \"src/winml/modelkit/loader/hf.py\": [\n [\n 204,\n 204\n ],\n [\n 206,\n 206\n ],\n [\n 0,\n 0\n ]\n ],\n \"src/winml/modelkit/serve/app.py\": [\n [\n 36,\n 37\n ],\n [\n 43,\n 43\n ],\n\ + \ [\n 216,\n 216\n ],\n [\n 218,\n 222\n ],\n [\n 289,\n 289\n ],\n [\n 322,\n 322\n ],\n [\n 369,\n 369\n ],\n [\n 374,\n 374\n ],\n [\n 376,\n 376\n ],\n [\n 379,\n 379\n ],\n [\n 382,\n 382\n ],\n [\n 393,\n 393\n ]\n ],\n \"src/winml/modelkit/serve/cli_api.py\": [\n [\n 33,\n 33\n ],\n [\n 39,\n 39\n ],\n [\n 147,\n 147\n ],\n [\n 149,\n 153\n ],\n [\n 180,\n 180\n ],\n [\n 251,\n 251\n ],\n [\n 293,\n 293\n ],\n [\n 365,\n 369\n ]\n ],\n \"src/winml/modelkit/serve/manager.py\": [\n [\n 52,\n 52\n ],\n [\n 57,\n 57\n ],\n [\n 113,\n 113\n ],\n [\n 251,\n 251\n ],\n [\n 262,\n 262\n ],\n [\n 439,\n 439\n ],\n [\n 452,\n 452\n ]\n ],\n \"src/winml/modelkit/serve/static/index.html\": [\n [\n 5,\n 5\n ],\n [\n 300,\n\ + \ 300\n ],\n [\n 920,\n 920\n ],\n [\n 945,\n 945\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ],\n [\n 0,\n 0\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"src/winml/modelkit/serve/cli_api.py::\",\n \"src/winml/modelkit/serve/app.py::create_app\"\n ],\n \"sites_in_set\": [\n \"src/winml/modelkit/serve/cli_api.py::\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"src/winml/modelkit/serve/cli_api.py\",\n \"\",\n 148\n ]\n ],\n \"kept\": [],\n \"triage\": {\n \"src/winml/modelkit/serve/cli_api.py::\": \"constant\"\n }\n}\n" diff --git a/plane/workspaces/winml-cli-serve-cors-vulnerable.yaml b/plane/workspaces/winml-cli-serve-cors-vulnerable.yaml index 7f795d6..f06c54e 100644 --- a/plane/workspaces/winml-cli-serve-cors-vulnerable.yaml +++ b/plane/workspaces/winml-cli-serve-cors-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/wp-directorist-avatar-ssrf-inputs.yaml b/plane/workspaces/wp-directorist-avatar-ssrf-inputs.yaml index bdde3d5..23f8a08 100644 --- a/plane/workspaces/wp-directorist-avatar-ssrf-inputs.yaml +++ b/plane/workspaces/wp-directorist-avatar-ssrf-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,7 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"directorist_rest_upload_image_from_url\",\n \"line\": 115,\n \"path\": \"includes/rest-api/functions.php\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-directorist-avatar-ssrf\",\n \"sites\": [\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.002616,\n \"recorded_usd\": 0.0352\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-directorist-avatar-ssrf\",\n \"ranges\": {\n \"config.php\": [\n [\n 4,\n 4\n ]\n ],\n \"directorist-base.php\": [\n [\n 6,\n 6\n ]\n ],\n \"includes/rest-api/Version1/class-orders-controller.php\": [\n [\n 84,\n 84\n ]\n ],\n \"includes/rest-api/Version1/class-users-controller.php\": [\n [\n 587,\n 589\n ],\n [\n 598,\n 598\n ],\n [\n 600,\n 601\n ],\n [\n 632,\n 632\n ]\n ],\n \"includes/rest-api/Version1/class-users-favorites-controller.php\": [\n [\n 131,\n 131\n ]\n ],\n \"includes/rest-api/functions.php\"\ + : [\n [\n 105,\n 105\n ],\n [\n 114,\n 114\n ],\n [\n 115,\n 115\n ],\n [\n 181,\n 181\n ]\n ],\n \"readme.txt\": [\n [\n 8,\n 8\n ],\n [\n 302,\n 302\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\"\n ],\n \"sites_in_set\": [\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"includes/rest-api/functions.php\",\n \"directorist_rest_upload_image_from_url\",\n 115\n ]\n ],\n \"kept\": [\n [\n \"includes/rest-api/functions.php\",\n \"directorist_rest_upload_image_from_url\",\n 115\n ]\n ],\n \"triage\": {\n \"includes/rest-api/functions.php::directorist_rest_upload_image_from_url\": \"parameter\"\n }\n}\n" diff --git a/plane/workspaces/wp-directorist-avatar-ssrf-vulnerable.yaml b/plane/workspaces/wp-directorist-avatar-ssrf-vulnerable.yaml index 45443d7..e13aa07 100644 --- a/plane/workspaces/wp-directorist-avatar-ssrf-vulnerable.yaml +++ b/plane/workspaces/wp-directorist-avatar-ssrf-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/wp-gopay-log-filter-sqli-inputs.yaml b/plane/workspaces/wp-gopay-log-filter-sqli-inputs.yaml index 88a8b59..528d62c 100644 --- a/plane/workspaces/wp-gopay-log-filter-sqli-inputs.yaml +++ b/plane/workspaces/wp-gopay-log-filter-sqli-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,6 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"\",\n \"line\": 35,\n \"path\": \"admin/views/log.php\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"injection\",\n \"id\": \"wp-gopay-log-filter-sqli\",\n \"sites\": [\n \"admin/views/log.php::\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.001375,\n \"recorded_usd\": 0.0113\n },\n \"family\": \"injection\",\n \"id\": \"wp-gopay-log-filter-sqli\",\n \"ranges\": {\n \"admin/views/log.php\": [\n [\n 12,\n 13\n ],\n [\n 15,\n 15\n ],\n [\n 17,\n 22\n ],\n [\n 30,\n 30\n ],\n [\n 36,\n 41\n ],\n [\n 104,\n 104\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"admin/views/log.php::\"\n ],\n \"sites_in_set\": [\n \"admin/views/log.php::\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"admin/views/log.php\",\n \"\",\n 35\n ]\n ],\n \"kept\": [\n [\n \"admin/views/log.php\",\n \"\",\n 35\n ]\n ],\n \"triage\": {\n \"admin/views/log.php::\": \"request\"\n }\n}\n" diff --git a/plane/workspaces/wp-gopay-log-filter-sqli-vulnerable.yaml b/plane/workspaces/wp-gopay-log-filter-sqli-vulnerable.yaml index 01451b8..486cc95 100644 --- a/plane/workspaces/wp-gopay-log-filter-sqli-vulnerable.yaml +++ b/plane/workspaces/wp-gopay-log-filter-sqli-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/plane/workspaces/wp-groundhogg-confirm-redirect-inputs.yaml b/plane/workspaces/wp-groundhogg-confirm-redirect-inputs.yaml index a509a60..f0c0826 100644 --- a/plane/workspaces/wp-groundhogg-confirm-redirect-inputs.yaml +++ b/plane/workspaces/wp-groundhogg-confirm-redirect-inputs.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: @@ -11,6 +11,6 @@ spec: - path: functions.json content: "[\n {\n \"function\": \"\",\n \"line\": 904,\n \"path\": \"templates/preferences.php\"\n }\n]\n" - path: case.json - content: "{\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-groundhogg-confirm-redirect\",\n \"sites\": [\n \"templates/preferences.php::\"\n ]\n}\n" + content: "{\n \"cost\": {\n \"candidates_before_triage\": 1,\n \"recorded_triage_basis\": \"derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded\",\n \"recorded_triage_usd\": 0.006524,\n \"recorded_usd\": 0.0585\n },\n \"family\": \"untrusted_destination\",\n \"id\": \"wp-groundhogg-confirm-redirect\",\n \"ranges\": {\n \"templates/preferences.php\": [\n [\n 904,\n 904\n ]\n ]\n },\n \"ranges_basis\": \"git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')\",\n \"sites\": [\n \"templates/preferences.php::\"\n ],\n \"sites_in_set\": [\n \"templates/preferences.php::\"\n ]\n}\n" - path: triage.json content: "{\n \"candidates\": [\n [\n \"templates/preferences.php\",\n \"\",\n 904\n ]\n ],\n \"kept\": [\n [\n \"templates/preferences.php\",\n \"\",\n 904\n ]\n ],\n \"triage\": {\n \"templates/preferences.php::\": \"request\"\n }\n}\n" diff --git a/plane/workspaces/wp-groundhogg-confirm-redirect-vulnerable.yaml b/plane/workspaces/wp-groundhogg-confirm-redirect-vulnerable.yaml index 2b2ced7..9849d9c 100644 --- a/plane/workspaces/wp-groundhogg-confirm-redirect-vulnerable.yaml +++ b/plane/workspaces/wp-groundhogg-confirm-redirect-vulnerable.yaml @@ -1,4 +1,4 @@ -# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --run validation-46`; do not edit, regenerate. +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. apiVersion: ax.io/v1alpha1 kind: Workspace metadata: diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 5ecf188..20cc372 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -256,6 +256,13 @@ def main(argv: list[str] | None = None) -> int: plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (--vulnerable_a.jsonl)") plane_ws.add_argument("--run", default="validation-46", help="with --validation-set: the Run name") plane_ws.add_argument("--plane", type=Path, default=Path("plane"), help="with --validation-set: the manifest root") + plane_ws.add_argument( + "--repos", type=Path, default=Path.home() / ".cache" / "openultrasast" / "independent", + help="with --validation-set: one clone per case id, for the fix ranges agree scores by (evaluate.hunks)", + ) # fmt: skip + plane_ws.add_argument( + "--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)" + ) args = parser.parse_args(argv) # Diagnosis needs the stage costs, and nothing configures logging, so the default root level of @@ -1324,11 +1331,18 @@ def _plane_increment(args: argparse.Namespace) -> int: print("--validation-set needs --candidates and --triage", file=sys.stderr) return 2 home = str(Path.home()) - shown = [str(p).replace(home, "~", 1) for p in (args.population, args.validation_set, args.candidates, args.triage)] - command = f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]} --run {args.run}" - written = increment( - args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command + paths = (args.population, args.validation_set, args.candidates, args.triage, args.repos) + shown = [str(p).replace(home, "~", 1) for p in paths] + command = ( + f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]}" + f" --repos {shown[4]} --run {args.run}" ) + if args.runner_image is not None: + command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}" + written = increment( + args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command, + repos=args.repos, runner_image=args.runner_image, + ) # fmt: skip print(f"{len(written)} manifests written under {args.plane} (Run {args.run})") return 0 diff --git a/src/openultrasast/plane/generate.py b/src/openultrasast/plane/generate.py index 1791af4..409af53 100644 --- a/src/openultrasast/plane/generate.py +++ b/src/openultrasast/plane/generate.py @@ -8,8 +8,11 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi ``openultrasast.io/git-commits`` annotation, as :mod:`.workspaces` writes it); - ``workspaces/-inputs.yaml``: a files-only Workspace with the inputs no task produces -- the recorded candidates (``candidates.json`` in the ``independent-v2-modelsinks`` shape for ``verify`` and ``agree``, - ``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), the case's declared sites - (``case.json`` for ``agree``) and the recorded triage (``triage.json``, for the measurement); + ``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), everything ``agree`` scores by + (``case.json``: the declared sites, the old-side fix ranges exactly as ``evaluate.hunks(case, "old")`` computes + them from ``--repos``, the declared sites among the set's candidates -- the reference's "of 20" -- and the + reference's cost: candidates before triage, recorded usd, and the triage share of it) and the recorded triage + (``triage.json``, for the measurement); - ``tasks/.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two Workspaces and naming the files as env; - ``runs/.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets. @@ -24,12 +27,21 @@ Candidates are the set's ``(path, function)`` pairs with the line of the case re ``benchmarks/independent/verify_batched.py`` does (sorted, the last line of a repeated pair wins), and filtered by the recorded triage (the ``kept`` lists of the batched-check scans): ``verify`` has no triage stage, so the hunts see exactly the candidates the reference hunts saw. Output is a pure function of the inputs. + +The recorded triage cost is derived per file: a batched-check record's ``usd`` paid for the triage call and the +hunts, and its ``usage`` counts only the hunts (``verify_sinks._Recording`` slices the client's usage from the +first hunt on), so the triage share is ``usd`` minus that usage priced at the reference model's list price. + +``--runner-image FILE`` (``~/.cache/ousast/ax-src/runner-image``, written by ``ops/ax/up.sh``) re-pins the +templates' ``image:`` to the digest in that file before generating, so a rebuilt image is one command. """ from __future__ import annotations import json import math +import re +import subprocess import tomllib from collections.abc import Mapping, Sequence from dataclasses import dataclass @@ -38,13 +50,21 @@ from typing import Any import yaml +from ..model.endpoint import price_of +from .budget import cost_of from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests from .reconciler import _ax_name, _doc from .tasks.verify import MAX_STEPS, units_of -__all__ = ["CaseInputs", "increment", "read_cases", "render"] +__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"] REFERENCE_USD_PER_CANDIDATE = 0.022 # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes +REFERENCE_MODEL = "deepseek-flash" # verify_batched.py's default (DEFAULT_DETECTOR_MODEL) in the batched check +TRIAGE_BASIS = ( + "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage " + f"priced at {REFERENCE_MODEL} list prices; the triage call's own usage was not recorded" +) +RANGES_BASIS = "git diff -U0 , old side: benchmarks/independent/evaluate.hunks(case, 'old')" HEADROOM = 1.5 CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content retry ATESPACE = "default" @@ -62,6 +82,37 @@ class CaseInputs: kept: tuple[Candidate, ...] triage: Mapping[str, str] recorded_usd: float + triage_usd: float = 0.0 + ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None + + +def fix_ranges(repo: Path, vulnerable: str, fixed: str) -> dict[str, list[tuple[int, int]]]: + """Changed line ranges per file on the vulnerable side of the fix: `evaluate.hunks(case, "old")`, line for + line (including its attribution of a hunk under ``--- /dev/null`` to the file before it).""" + if not (repo / ".git").exists() and not (repo / "HEAD").is_file(): + raise ValueError(f"no repository at {repo}: the fix ranges cannot be computed") + diff = subprocess.run(["git", "-C", str(repo), "diff", "-U0", vulnerable, fixed], capture_output=True, text=True, check=True).stdout + ranges: dict[str, list[tuple[int, int]]] = {} + current = "" + for line in diff.splitlines(): + if line.startswith("--- a/"): + current = line[6:] + elif line.startswith("@@") and current: + match = re.search(r"-(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?", line) + if match: + first, length = int(match.group(1)), int(match.group(2)) if match.group(2) is not None else 1 + ranges.setdefault(current, []).append((first, first + max(length, 1) - 1)) + if not ranges: + raise ValueError(f"{repo}: the diff {vulnerable[:12]}..{fixed[:12]} changed no line on the vulnerable side") + return ranges + + +def _triage_usd(rows: Sequence[Mapping[str, Any]]) -> float: + """The triage share of the recorded spend: per file, usd minus the recorded hunt usage at list price.""" + prices = price_of(REFERENCE_MODEL) + if prices is None: + raise ValueError(f"no recorded price for {REFERENCE_MODEL}") + return round(sum(max(float(r.get("usd") or 0.0) - cost_of(r.get("usage") or {}, prices), 0.0) for r in rows), 6) def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]: @@ -75,8 +126,11 @@ def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]: return rows -def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path) -> list[CaseInputs]: - """The set's cases in id order, each with its candidates, the triage-kept subset and the recorded cost.""" +def read_cases( + validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None +) -> list[CaseInputs]: + """The set's cases in id order, each with its candidates, the triage-kept subset, the recorded cost and its + triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges.""" subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8")) records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]} cases: list[CaseInputs] = [] @@ -104,6 +158,8 @@ def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, pop kept=tuple(c for c in chosen if (c[0], c[1]) in kept_keys), triage=dict(sorted(triage.items())), recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6), + triage_usd=_triage_usd(rows), + ranges=None if repos is None else fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"]), ) ) return cases @@ -117,6 +173,23 @@ def _json(payload: object) -> str: return json.dumps(payload, indent=1, sort_keys=True) + "\n" +def _case_record(item: CaseInputs) -> dict[str, Any]: + """What `agree` scores by: sites, fix ranges (absent without ``--repos``), sites in the set, reference cost.""" + case = item.case + sites = [str(s) for s in case.get("sites", [])] + in_set = {f"{p}::{fn}" for p, fn, _ in item.candidates} + record: dict[str, Any] = {"id": case["id"], "family": case["family"], "sites": sites} + if item.ranges is not None: + record["ranges"] = {path: [list(span) for span in spans] for path, spans in item.ranges.items()} + record["ranges_basis"] = RANGES_BASIS + record["sites_in_set"] = [s for s in sites if s in in_set] + record["cost"] = { + "candidates_before_triage": len(item.candidates), "recorded_usd": item.recorded_usd, + "recorded_triage_usd": item.triage_usd, "recorded_triage_basis": TRIAGE_BASIS, + } # fmt: skip + return record + + def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]: case = item.case git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}] @@ -125,7 +198,7 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]: files = { "candidates.json": _json({"id": case["id"], "family": case["family"], "candidates": [list(c) for c in item.kept]}), "functions.json": _json([{"path": p, "function": fn, "line": ln} for p, fn, ln in item.kept]), - "case.json": _json({"id": case["id"], "family": case["family"], "sites": list(case.get("sites", []))}), + "case.json": _json(_case_record(item)), "triage.json": _json( {"candidates": [list(c) for c in item.candidates], "kept": [list(c) for c in item.kept], "triage": dict(item.triage)} ), @@ -207,12 +280,43 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: return out +_IMAGE_LINE = re.compile(r'^(\s*image:\s*)"?[^"\s]+"?\s*$', re.MULTILINE) + + +def repin_templates(plane: Path, image_file: Path) -> str: + """Set every template's ``image:`` to the digest-pinned reference in ``image_file`` (as ``ops/ax/up.sh`` + writes it); comments and layout are kept. Returns the reference.""" + image = image_file.read_text(encoding="utf-8").strip() + if not re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", image): + raise ValueError(f"{image_file}: expected a digest-pinned image reference, got {image!r}") + for name in TEMPLATES: + path = plane / "tasks" / f"{name}.yaml" + text = path.read_text(encoding="utf-8") + pinned, count = _IMAGE_LINE.subn(lambda m: f'{m.group(1)}"{image}"', text) + if count != 1: + raise ValueError(f"{path}: {count} image lines, expected one") + path.write_text(pinned, encoding="utf-8") + return image + + def increment( - population: Path, validation_set: Path, candidates_dir: Path, triage_dir: Path, *, plane: Path, run_name: str, command: str + population: Path, + validation_set: Path, + candidates_dir: Path, + triage_dir: Path, + *, + plane: Path, + run_name: str, + command: str, + repos: Path | None = None, + runner_image: Path | None = None, ) -> list[Path]: - """Write the generated manifests under ``plane`` from the templates in ``plane/tasks``; returns the paths.""" + """Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when + ``runner_image`` is given); returns the paths.""" + if runner_image is not None: + repin_templates(plane, runner_image) loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES]) - files = render(read_cases(validation_set, candidates_dir, triage_dir, population), loaded.tasks, run_name, command) + files = render(read_cases(validation_set, candidates_dir, triage_dir, population, repos), loaded.tasks, run_name, command) written: list[Path] = [] for relative, text in files.items(): path = plane / relative diff --git a/src/openultrasast/plane/tasks/agree.py b/src/openultrasast/plane/tasks/agree.py index 9abc873..ef0ce3d 100644 --- a/src/openultrasast/plane/tasks/agree.py +++ b/src/openultrasast/plane/tasks/agree.py @@ -3,31 +3,65 @@ The rule is `verify_batched.agreement`: a candidate is reported when every pass flagged it; a candidate flagged by one pass is `disputed`. Beside the verdicts go the measurement's numbers: per-candidate cost (a hunt's usd split evenly over the candidates it asked about, summed over the passes), tool turns per hunt, and -- when the -case's declared sites are given -- the site match by `benchmarks/independent/evaluate.matches_v2`, the rule -protocol v2 scores by, imported from the benchmark tree rather than copied. +case's declared sites and fix ranges are given -- the site match by :func:`matches_declared`, which is +`benchmarks/independent/evaluate.matches_v2` (the rule protocol v2 and `score_batched.py` score by) reproduced +here so the runner image needs no benchmark tree; `tests/test_plane_scoring_parity.py` proves identical verdicts +on the recorded reference outputs. The ranges are the reference's own (`evaluate.hunks(case, "old")`), resolved +by the generator into `case.json`, and sites are anchored to the candidate as `score_batched.anchored` does. + +Cost is reported twice: `cost_per_candidate` is the plane's spend alone; `cost_per_candidate_with_recorded_triage` +adds the recorded triage cost of the case (`case.json` `cost.recorded_triage_usd`), because the reference's +$0.022 per candidate for two passes paid for triage and the plane applies the recorded triage instead. Both +divide by the candidates before triage when `case.json` gives that count, as the reference did. Env contract: `OUSAST_INPUT_PASS_A`, `OUSAST_INPUT_PASS_B` (a verify output directory or its `units.jsonl`), -`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (a JSON case record with `family` -and `sites` in the `population-v2.toml` shape, or that TOML itself with `OUSAST_CASE_ID` naming the case) and -`OUSAST_BENCHMARKS_DIR` (where `independent/evaluate.py` lives when the package is not run from the repository). -Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished. +`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (the generator's `case.json`: +`family`, `sites`, `ranges`, `sites_in_set`, `cost`; or a `population-v2.toml`-shaped TOML with `OUSAST_CASE_ID` +naming the case). Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished. """ from __future__ import annotations -import importlib.util import json import os import sys import tomllib import traceback -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any from .verify import EXIT_CODES, USAGE_FIELDS, load_candidates, read_units -Matcher = Callable[[dict[str, Any], dict[str, Any], dict[str, list[tuple[int, int]]]], bool] +LINE_WINDOW = 10 # evaluate.LINE_WINDOW: a finding within 10 lines of a changed range matches + + +def matches_declared(finding: Mapping[str, Any], case: Mapping[str, Any], ranges: Mapping[str, Sequence[Sequence[int]]]) -> bool: + """`evaluate.matches_v2`: the finding's family is the case's, and its `path:line:function` site is a declared + `path::function`, or lies in a file the fix changed and names a declared site's function or is within + `LINE_WINDOW` lines of a changed range (`ranges`: the old side of the fix diff, per file).""" + path, _, rest = str(finding["site"]).partition(":") + line_text, _, function = rest.partition(":") + if finding.get("family") != case["family"]: + return False + sites = {str(s) for s in case.get("sites", [])} + if f"{path}::{function}" in sites: + return True + if path not in ranges: + return False + if function in {site.partition("::")[2] for site in sites}: + return True + line = int(line_text) if line_text.isdigit() else -1 + return any(int(first) - LINE_WINDOW <= line <= int(last) + LINE_WINDOW for first, last in ranges[path]) + + +def anchor_site(candidate: str, site: str) -> str: + """`score_batched.anchored`: the site is the candidate's file and function; the reported line is kept only + when the site is in the candidate's file (otherwise 0).""" + path, _, function = candidate.partition("::") + site_path, _, rest = site.partition(":") + line = rest.split(":")[0] if site_path == path else "0" + return f"{path}:{line}:{function}" def agreement(passes: Sequence[Sequence[Mapping[str, Any]]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: @@ -51,7 +85,8 @@ def load_pass(source: Path) -> list[dict[str, Any]]: def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None: - """A case record with `family`, `sites` and optional `ranges` (`{path: [[first, last], ...]}`).""" + """A case record with `family`, `sites` and `ranges` (`{path: [[first, last], ...]}`); without `ranges` + site matching is reported as not assessed.""" if path is None: return None if path.suffix == ".toml": @@ -66,24 +101,6 @@ def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None: return record -def load_matcher(benchmarks_dir: Path | None = None) -> Matcher | None: - """`evaluate.matches_v2` from the benchmark tree, loaded by file so the package never imports it by name; - None when the tree is not present (site matching is then reported as not assessed, never as a miss).""" - roots = [benchmarks_dir] if benchmarks_dir is not None else [] - roots += [p / "benchmarks" for p in (Path.cwd(), *Path(__file__).resolve().parents)] - for root in roots: - script = root / "independent" / "evaluate.py" - if script.is_file(): - spec = importlib.util.spec_from_file_location("ousast_benchmark_evaluate", script) - if spec is None or spec.loader is None: - continue - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - matcher: Matcher = module.matches_v2 - return matcher - return None - - def _per_candidate_cost(rows: Sequence[Mapping[str, Any]]) -> tuple[dict[str, float | None], dict[str, int]]: """A hunt's usd split evenly over its candidates; its turns and calls counted once per candidate.""" usd: dict[str, float | None] = {} @@ -101,6 +118,10 @@ def _mean(values: Sequence[float]) -> float | None: return round(sum(values) / len(values), 3) if values else None +def _per_candidate(total: float | None, count: int) -> float | None: + return None if total is None or not count else round(total / count, 6) + + def run( output_dir: Path, candidates: object, @@ -108,7 +129,6 @@ def run( pass_b: Sequence[Mapping[str, Any]], *, sites: Mapping[str, Any] | None = None, - matcher: Matcher | None = None, ) -> dict[str, Any]: output_dir.mkdir(parents=True, exist_ok=True) family, unique = load_candidates(candidates) @@ -121,18 +141,17 @@ def run( case = dict(sites) if sites else None if case is not None and "family" not in case and family: case["family"] = family - ranges = {p: [(int(a), int(b)) for a, b in spans] for p, spans in (case or {}).get("ranges", {}).items()} - match_state = ( - "not assessed: no sites given" if case is None else ("not assessed: evaluate.py not found" if matcher is None else "assessed") - ) + ranges = (case or {}).get("ranges") + assessed = case is not None and isinstance(ranges, Mapping) + match_state = "assessed" if assessed else ("not assessed: no sites given" if case is None else "not assessed: no fix ranges given") rows: list[dict[str, Any]] = [] for name in names: in_a, in_b = (name in flagged[0] if name in asked[0] else None), (name in flagged[1] if name in asked[1] else None) finding = flagged[0].get(name) or flagged[1].get(name) - site = finding["site"] if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}" + site = anchor_site(name, finding["site"]) if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}" site_match = None - if case is not None and matcher is not None: - site_match = matcher({"site": site, "family": (finding or {}).get("family", family)}, case, ranges) + if assessed and case is not None and isinstance(ranges, Mapping): + site_match = matches_declared({"site": site, "family": (finding or {}).get("family", family)}, case, ranges) cost = usd_by.get(name) rows.append({ "candidate": name, "site": site, "a": in_a, "b": in_b, "agreed": bool(in_a and in_b), "disputed": bool(in_a) != bool(in_b), @@ -142,12 +161,20 @@ def run( declared = [r for r in rows if r["site_match"]] hunts = [*pass_a, *pass_b] usd_total = None if any(r.get("usd") is None for r in hunts) else round(sum(float(r["usd"]) for r in hunts), 6) + reference: Mapping[str, Any] = (case or {}).get("cost") or {} + before = int(reference.get("candidates_before_triage") or len(names)) + triage_usd = float(reference["recorded_triage_usd"]) if reference.get("recorded_triage_usd") is not None else None + with_triage = None if usd_total is None or triage_usd is None else round(usd_total + triage_usd, 6) metrics = { - "candidates": len(names), "asked_in_both": len(names) - len(unasked), "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]), - "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts), - "declared_sites": len(set((case or {}).get("sites", []))), "declared_sites_matched": len(declared), - "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]), "site_matching": match_state, - "usd_total": usd_total, "cost_per_candidate": (None if usd_total is None or not names else round(usd_total / len(names), 6)), + "candidates": len(names), "candidates_before_triage": before, "asked_in_both": len(names) - len(unasked), + "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]), "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts), + "declared_sites": len(set((case or {}).get("sites", []))), + "declared_sites_in_set": (len(case["sites_in_set"]) if case and "sites_in_set" in case else None), + "declared_sites_matched": len(declared), "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]), + "site_matching": match_state, + "usd_total": usd_total, "cost_per_candidate": _per_candidate(usd_total, before), + "recorded_triage_usd": triage_usd, "recorded_triage_basis": reference.get("recorded_triage_basis"), + "usd_total_with_recorded_triage": with_triage, "cost_per_candidate_with_recorded_triage": _per_candidate(with_triage, before), "turns_per_hunt": _mean([float(r.get("turns") or 0) for r in hunts]), "calls_per_hunt": _mean([float((r.get("usage") or {}).get("calls") or 0) for r in hunts]), } # fmt: skip @@ -174,8 +201,7 @@ def main() -> int: candidates = json.loads(Path(os.environ["OUSAST_INPUT_CANDIDATES"]).read_text()) sites_path = Path(os.environ["OUSAST_INPUT_SITES"]) if os.environ.get("OUSAST_INPUT_SITES") else None sites = load_sites(sites_path, os.environ.get("OUSAST_CASE_ID")) - benchmarks = Path(os.environ["OUSAST_BENCHMARKS_DIR"]) if os.environ.get("OUSAST_BENCHMARKS_DIR") else None - summary = run(output_dir, candidates, pass_a, pass_b, sites=sites, matcher=load_matcher(benchmarks) if sites else None) + summary = run(output_dir, candidates, pass_a, pass_b, sites=sites) except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback summary = { "status": "failed", "units_done": 0, "units_total": 0, "usd": 0, "calls": 0, "usage": dict.fromkeys(USAGE_FIELDS, 0), @@ -188,7 +214,7 @@ def main() -> int: return EXIT_CODES.get(str(summary["status"]), 2) -__all__ = ["agreement", "load_matcher", "load_pass", "load_sites", "main", "run"] +__all__ = ["LINE_WINDOW", "agreement", "anchor_site", "load_pass", "load_sites", "main", "matches_declared", "run"] if __name__ == "__main__": diff --git a/tests/test_plane_increment.py b/tests/test_plane_increment.py index 95d6db9..d59c46a 100644 --- a/tests/test_plane_increment.py +++ b/tests/test_plane_increment.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import re import shutil +import subprocess import tomllib from pathlib import Path @@ -15,13 +16,14 @@ from test_plane_reconciler import AX_FIELDS, non_ax_fields from openultrasast.model.endpoint import price_of from openultrasast.plane import reconciler from openultrasast.plane.budget import prices_from -from openultrasast.plane.generate import TEMPLATES, increment, read_cases, render +from openultrasast.plane.generate import TEMPLATES, fix_ranges, increment, read_cases, render, repin_templates from openultrasast.plane.manifests import load_manifests PLANE = Path("plane") RUN = PLANE / "runs" / "validation-46.yaml" POPULATION = Path("benchmarks/independent/population-v2.toml") RESULTS = Path.home() / "ousast-results" +REPOS = Path.home() / ".cache" / "openultrasast" / "independent" PIN = "0123456789abcdef0123456789abcdef01234567" @@ -81,14 +83,30 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None: def test_inputs_carry_the_validation_set() -> None: manifests = load_manifests(sorted((PLANE / "workspaces").glob("*-inputs.yaml"))) assert len(manifests.workspaces) == 15 - candidates = kept = 0 + candidates = kept = in_set = before = 0 + usd = triage = 0.0 for workspace in manifests.workspaces.values(): files = {f.path: json.loads(f.content) for f in workspace.files} assert set(files) == {"candidates.json", "functions.json", "case.json", "triage.json"} candidates += len(files["triage.json"]["candidates"]) kept += len(files["candidates.json"]["candidates"]) assert [[r["path"], r["function"], r["line"]] for r in files["functions.json"]] == files["candidates.json"]["candidates"] - assert (candidates, kept) == (46, 43) + case = files["case.json"] + assert case["ranges"] and all(first <= last for spans in case["ranges"].values() for first, last in spans), case["id"] + assert set(case["sites_in_set"]) <= set(case["sites"]) + in_set += len(case["sites_in_set"]) + before += case["cost"]["candidates_before_triage"] + usd += case["cost"]["recorded_usd"] + triage += case["cost"]["recorded_triage_usd"] + assert 0 < case["cost"]["recorded_triage_usd"] <= case["cost"]["recorded_usd"], case["id"] + assert (candidates, kept, in_set, before) == (46, 43, 20, 46) + # the reference's $1.01 for 46 candidates, two passes, triage included; triage was about 9% of it + assert round(usd / before, 3) == 0.022 and 0.05 < triage / usd < 0.15 + + +def test_templates_are_pinned_to_one_runner_image() -> None: + images = {task.image for task in load_manifests(sorted((PLANE / "tasks").glob("*.yaml"))).tasks.values()} + assert len(images) == 1 and re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", images.pop() or "") def _committed_command() -> str: @@ -98,7 +116,9 @@ def _committed_command() -> str: return found.group(1) -@pytest.mark.skipif(not (RESULTS / "independent-v2-batched-check" / "scans").is_dir(), reason="recorded results not on this host") +@pytest.mark.skipif( + not (RESULTS / "independent-v2-batched-check" / "scans").is_dir() or not REPOS.is_dir(), reason="recorded results not on this host" +) def test_committed_manifests_match_the_generator() -> None: templates = load_manifests([PLANE / "tasks" / f"{name}.yaml" for name in TEMPLATES]).tasks cases = read_cases( @@ -106,6 +126,7 @@ def test_committed_manifests_match_the_generator() -> None: RESULTS / "independent-v2-modelsinks", RESULTS / "independent-v2-batched-check" / "scans", POPULATION, + REPOS, ) for relative, text in render(cases, templates, "validation-46", _committed_command()).items(): assert (PLANE / relative).read_text() == text, relative @@ -173,3 +194,43 @@ def test_generator_refuses_a_set_the_recorded_scans_did_not_ask(tmp_path: Path) (paths["scans"] / "alpha--vulnerable_a.jsonl").unlink() with pytest.raises(ValueError, match="no recorded triage"): _generate(paths, paths["plane"]) + + +def _git(repo: Path, *args: str) -> str: + done = subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t", *args], capture_output=True, text=True, check=True + ) + return done.stdout.strip() + + +def test_fix_ranges_are_the_old_side_of_the_fix_diff(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q") + (repo / "a.py").write_text("".join(f"line {i}\n" for i in range(1, 31))) + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "vulnerable") + vulnerable = _git(repo, "rev-parse", "HEAD") + (repo / "a.py").write_text("".join(f"line {i}\n" if i not in (3, 4, 20) else "fixed\n" for i in range(1, 31)) + "added\n") + _git(repo, "commit", "-qam", "fixed") + assert fix_ranges(repo, vulnerable, _git(repo, "rev-parse", "HEAD")) == {"a.py": [(3, 4), (20, 20), (30, 30)]} + with pytest.raises(ValueError, match="no repository"): + fix_ranges(tmp_path / "absent", vulnerable, vulnerable) + + +def test_case_record_carries_sites_in_set_and_cost_and_the_image_is_repinned(tmp_path: Path) -> None: + paths = _fixture(tmp_path) + digest = "localhost:5001/ousast-runner@sha256:" + "ab" * 32 + (tmp_path / "runner-image").write_text(digest + "\n") + increment( + paths["population"], paths["set"], paths["candidates"], paths["scans"], plane=paths["plane"], run_name="check", + command="ousast plane test", runner_image=tmp_path / "runner-image", + ) # fmt: skip + manifests = load_manifests(sorted(paths["plane"].rglob("*.yaml"))) + assert {t.image for t in manifests.tasks.values()} == {digest} + case = json.loads(next(f.content for f in manifests.workspaces["alpha-inputs"].files if f.path == "case.json")) + assert "ranges" not in case and case["sites_in_set"] == ["a.py::run"] # no --repos: agree reports not assessed + assert case["cost"]["candidates_before_triage"] == 2 and case["cost"]["recorded_triage_usd"] == 0.5 # no hunt usage recorded + (tmp_path / "runner-image").write_text("localhost:5001/ousast-runner:dev\n") + with pytest.raises(ValueError, match="digest-pinned"): + repin_templates(paths["plane"], tmp_path / "runner-image") diff --git a/tests/test_plane_scoring_parity.py b/tests/test_plane_scoring_parity.py new file mode 100644 index 0000000..0093578 --- /dev/null +++ b/tests/test_plane_scoring_parity.py @@ -0,0 +1,144 @@ +"""`agree` scores exactly as the reference measurement did (ai-service-plane Requirement 6). + +The reference is `benchmarks/independent/score_batched.py` over the batched-check scans: sites anchored by +`score_batched.anchored`, matched by `evaluate.matches_v2` against `evaluate.hunks(case, "old")`. The plane cannot +ship the benchmark tree, so the generator resolves the ranges into `case.json` and `agree.matches_declared` +reproduces the rule. These tests run both on the recorded reference outputs of the 15 validation cases and +require identical verdicts, identical ranges, and the reference's own numbers (16 of 20, $0.022) back from `agree`. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import tomllib +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from openultrasast.plane.manifests import load_manifests +from openultrasast.plane.tasks import agree + +BENCH = Path("benchmarks/independent") +POPULATION = BENCH / "population-v2.toml" +SCANS = Path.home() / "ousast-results" / "independent-v2-batched-check" / "scans" +REPOS = Path.home() / ".cache" / "openultrasast" / "independent" +INPUTS = sorted(Path("plane/workspaces").glob("*-inputs.yaml")) + +pytestmark = pytest.mark.skipif( + not SCANS.is_dir() or not REPOS.is_dir(), reason="recorded batched-check scans or case clones not on this host" +) + + +def _load(name: str) -> ModuleType: + sys.path.insert(0, str(BENCH.resolve())) # score_batched imports evaluate by name + try: + spec = importlib.util.spec_from_file_location(f"parity_{name}", BENCH / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + sys.path.remove(str(BENCH.resolve())) + + +@pytest.fixture(scope="module") +def reference() -> tuple[ModuleType, ModuleType]: + evaluate, score_batched = _load("evaluate"), _load("score_batched") + evaluate.use(POPULATION.resolve()) + return evaluate, score_batched + + +def _inputs() -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for workspace in load_manifests(INPUTS).workspaces.values(): + files = {f.path: json.loads(f.content) for f in workspace.files} + out[files["case.json"]["id"]] = files + assert len(out) == 15, sorted(out) + return out + + +def _records() -> dict[str, dict[str, Any]]: + return {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]} + + +def test_committed_ranges_are_the_reference_hunks(reference: tuple[ModuleType, ModuleType]) -> None: + evaluate, _ = reference + records = _records() + for case_id, files in _inputs().items(): + hunks = {path: [list(span) for span in spans] for path, spans in evaluate.hunks(records[case_id], "old").items()} + assert files["case.json"]["ranges"] == hunks, case_id + + +def test_matching_verdicts_are_identical_on_the_recorded_outputs(reference: tuple[ModuleType, ModuleType]) -> None: + evaluate, score_batched = reference + records = _records() + compared = declared = matched = 0 + for case_id, files in _inputs().items(): + record, mine = records[case_id], files["case.json"] + old = evaluate.hunks(record, "old") + recorded = json.loads((SCANS / f"{case_id}--vulnerable_a.json").read_text()) + raw = [dict(f) for key in ("findings", "disputed") for f in recorded.get(key, [])] + anchored = score_batched.anchored(json.loads(json.dumps({"findings": raw})))["findings"] + assert [agree.anchor_site(f["candidate"], f["site"]) for f in raw] == [f["site"] for f in anchored], case_id + family = record["family"] + probes = [{"site": f["site"], "family": f["family"]} for f in anchored] + probes += [{"site": f"{p}:{ln}:{fn}", "family": family} for p, fn, ln in files["triage.json"]["candidates"]] + sites = [{"site": f"{s.partition('::')[0]}:0:{s.partition('::')[2]}", "family": family} for s in record.get("sites", [])] + declared += len(sites) + probes += sites + probes += [dict(p, family="not-" + family) for p in probes] + for probe in probes: + theirs = evaluate.matches_v2(probe, record, old) + assert agree.matches_declared(probe, mine, mine["ranges"]) == theirs, (case_id, probe) + compared += 1 + matched += theirs + assert declared == 22 and compared > 2 * (declared + 46) + assert matched > 0 + + +def _units(case_id: str, kept: list[list[Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], float]: + """The recorded per-file hunts as two `verify` passes: a hunt's usd is the file's spend minus its triage.""" + from openultrasast.model.endpoint import price_of + from openultrasast.plane.budget import cost_of + + prices = price_of("deepseek-flash") + assert prices is not None + passes: tuple[list[dict[str, Any]], list[dict[str, Any]]] = ([], []) + triage = 0.0 + for text in (SCANS / f"{case_id}--vulnerable_a.jsonl").read_text().splitlines(): + row = json.loads(text) + hunts = cost_of(row.get("usage") or {}, prices) + triage += max(float(row["usd"]) - hunts, 0.0) + asked = [c for c in kept if c[0] == row["path"]] + if not asked: + continue + for index, flagged in enumerate(row["passes"]): + passes[index].append({"path": row["path"], "candidates": asked, "flagged": flagged, "usd": hunts / 2, "turns": 0}) + return passes[0], passes[1], triage + + +def test_agree_on_the_reference_outputs_reproduces_the_reference_numbers(tmp_path: Path) -> None: + score = json.loads((SCANS.parent / "score.json").read_text()) + detected = {c["id"]: len(c["detected_sites"]) for c in score["cases"] if "detected_sites" in c and c["detected_sites"] is not None} + agreed = in_set = before = 0 + spend = with_triage = 0.0 + for case_id, files in _inputs().items(): + case = files["case.json"] + pass_a, pass_b, triage = _units(case_id, files["candidates.json"]["candidates"]) + assert case["cost"]["recorded_triage_usd"] == pytest.approx(triage, abs=1e-6) + summary = agree.run(tmp_path / case_id, files["candidates.json"], pass_a, pass_b, sites=case) + metrics = summary["metrics"] + assert metrics["site_matching"] == "assessed" + assert metrics["declared_sites_matched_agreed"] == detected[case_id], case_id + agreed += metrics["declared_sites_matched_agreed"] + in_set += metrics["declared_sites_in_set"] + before += metrics["candidates_before_triage"] + spend += metrics["usd_total"] + with_triage += metrics["usd_total_with_recorded_triage"] + assert (agreed, in_set, before) == (16, 20, 46) + assert with_triage == pytest.approx(1.0123, abs=1e-3) # the recorded spend: the reference's $1.01 + assert round(with_triage / before, 3) == 0.022 and spend < with_triage diff --git a/tests/test_plane_verify.py b/tests/test_plane_verify.py index ff7e724..f1f8d98 100644 --- a/tests/test_plane_verify.py +++ b/tests/test_plane_verify.py @@ -230,8 +230,8 @@ def _two_passes(tmp_path: Path) -> tuple[Path, Path, Path]: def test_agree_over_two_scripted_passes_computes_agreement_and_metrics(tmp_path: Path) -> None: _, a, b = _two_passes(tmp_path) out = tmp_path / "agree" - sites = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"]} - summary = agree.run(out, CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=sites, matcher=agree.load_matcher()) + sites = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"], "ranges": {"app.py": [[40, 41]]}} + summary = agree.run(out, CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=sites) assert summary["status"] == "done" and summary["usd"] == 0 and summary["calls"] == 0 and summary["model"] is None report = json.loads((out / "agreed.json").read_text()) by_name = {r["candidate"]: r for r in report["candidates"]} @@ -255,6 +255,34 @@ def test_agree_over_two_scripted_passes_computes_agreement_and_metrics(tmp_path: assert by_name["app.py::run"]["usd"] == pytest.approx(2 * per_call, abs=1e-6) +def test_agree_reports_cost_with_the_recorded_triage_and_needs_fix_ranges_to_assess_sites(tmp_path: Path) -> None: + _, a, b = _two_passes(tmp_path) + cost = {"candidates_before_triage": 4, "recorded_usd": 1.0, "recorded_triage_usd": 0.01, "recorded_triage_basis": "derived"} + sites = {"id": "case", "family": "injection", "sites": ["app.py::run"], "sites_in_set": ["app.py::run"], "cost": cost} + metrics = agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=sites)["metrics"] + spend = metrics["usd_total"] + assert metrics["candidates"] == 3 and metrics["candidates_before_triage"] == 4 and metrics["declared_sites_in_set"] == 1 + assert metrics["cost_per_candidate"] == pytest.approx(spend / 4, abs=1e-6) + assert metrics["usd_total_with_recorded_triage"] == pytest.approx(spend + 0.01, abs=1e-6) + assert metrics["cost_per_candidate_with_recorded_triage"] == pytest.approx((spend + 0.01) / 4, abs=1e-6) + # without the fix ranges a miss cannot be told from a match: not assessed, never a silent zero + assert metrics["site_matching"] == "not assessed: no fix ranges given" and metrics["declared_sites_matched"] == 0 + + +def test_matches_declared_is_the_protocol_v2_rule() -> None: + case = {"family": "injection", "sites": ["a.py::run", "b.py::"]} + ranges = {"a.py": [[50, 52]], "c.py": [[5, 5]]} + hit = lambda site, family="injection": agree.matches_declared({"site": site, "family": family}, case, ranges) # noqa: E731 + assert hit("a.py:0:run") and hit("b.py:0:") # a declared path::function, whether or not the fix touched it + assert not hit("a.py:0:run", "ssrf") # another family never matches + assert hit("a.py:62:helper") and not hit("a.py:63:helper") and hit("a.py:40:x") and not hit("a.py:39:x") # 10-line window + assert hit("c.py:900:run") and not hit("d.py:1:run") # a declared function name in a changed file only + assert ( + not hit("a.py:x:helper") and hit("c.py:x:helper") and agree.anchor_site("a.py::run", "other.py:7:run") == "a.py:0:run" + ) # a non-numeric line is -1, as in the reference + assert agree.anchor_site("a.py::run", "a.py:7:renamed") == "a.py:7:run" + + def test_agree_without_sites_leaves_site_matching_unassessed_and_flags_unasked_candidates(tmp_path: Path) -> None: _, a, b = _two_passes(tmp_path) rows_b = [r for r in agree.load_pass(b) if r["path"] != "db.py"] @@ -269,7 +297,8 @@ def test_agree_main_reads_the_env_contract(tmp_path: Path, monkeypatch: pytest.M _, a, b = _two_passes(tmp_path) (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES)) (tmp_path / "case.toml").write_text( - '[[case]]\nid = "case"\nfamily = "injection"\nsites = ["app.py::run"]\n\n[[case]]\nid = "x"\nfamily = "injection"\nsites = []\n' + '[[case]]\nid = "case"\nfamily = "injection"\nsites = ["app.py::run"]\nranges = { "app.py" = [[40, 41]] }\n\n' + '[[case]]\nid = "x"\nfamily = "injection"\nsites = []\n' ) env = { "OUSAST_OUTPUT_DIR": str(tmp_path / "agree"), "OUSAST_INPUT_PASS_A": str(a), "OUSAST_INPUT_PASS_B": str(b / "units.jsonl"),