diff --git a/.kiro/specs/ai-service-plane/design.md b/.kiro/specs/ai-service-plane/design.md index 5c104da..83ee4eb 100644 --- a/.kiro/specs/ai-service-plane/design.md +++ b/.kiro/specs/ai-service-plane/design.md @@ -206,6 +206,11 @@ plane applies the recorded triage without a model call, so `cost_per_candidate` not judged; the triage share is derived per file from the recorded spend minus the priced hunt usage (about $0.087 of the $1.01) and labelled as derived in `case.json`. Sites are judged as `declared_sites_matched_agreed` against `declared_sites_in_set`, the reference's 16 of 20. +The tie-break (Requirement 6.4) appends per case `-vc`, `verify` pass c restricted by `OUSAST_INPUT_ONLY` +to the agree task's `disputed.json` (an empty list is `done` with no model call), and `-final`, `agree` over +passes a, b and c, which decides 2-of-3 where pass c asked and keeps the a/b rule elsewhere, so pass c alone never +decides and `metrics.tiebreak` counts the flips; the first 60 entries stay byte-identical, so a rerun keeps their +`done` state, executes only the 30 new tasks, and pass c's cost adds to `usd_total` for the 6.2 judgement. ## Data Models diff --git a/.kiro/specs/ai-service-plane/requirements.md b/.kiro/specs/ai-service-plane/requirements.md index 3c7c8bb..0378873 100644 --- a/.kiro/specs/ai-service-plane/requirements.md +++ b/.kiro/specs/ai-service-plane/requirements.md @@ -132,6 +132,9 @@ before the remaining concerns are converted. current $0.022 for two passes; otherwise the increment is reported as not met and the design revisited. 3. The development transcript measure (`benchmarks/dev/token_report.py`) is run at the end of the increment and recorded next to the result. +4. After the first measurement (plane-increment-1.json: 14/20 agreed, 19/20 in either pass), agreement is decided + 2-of-3 with a third pass run only on candidates passes a and b dispute; the gate of 6.2 is re-evaluated on that + decision, and the result is recorded as plane-increment-2.json next to the first. ### Requirement 7: Routing is a declared concern of the plane diff --git a/.kiro/specs/ai-service-plane/tasks.md b/.kiro/specs/ai-service-plane/tasks.md index 3c61ba4..91bfb27 100644 --- a/.kiro/specs/ai-service-plane/tasks.md +++ b/.kiro/specs/ai-service-plane/tasks.md @@ -130,6 +130,15 @@ mypy are clean, and the commit is gated on pytest's own exit code. Script-level if not met, revise the design before task 9. - _Requirements: 6.1, 6.2, 6.3_ +- [ ] 8.1 Tie-break pass and re-measurement + - `verify` takes `OUSAST_INPUT_ONLY` (pass c on the disputed candidates only); `agree` takes + `OUSAST_INPUT_PASS_C` and decides 2-of-3 where pass c asked, the a/b rule elsewhere (`metrics.tiebreak`). + - `validation-46` gains `-vc` and `-final` per case, appended so the first 60 entries keep their + state; an offline replay with an empty pass c reproduces the recorded 14/20. + - Rerun `validation-46` on ax (only the new tasks run), re-evaluate the 6.2 gate on the final decision and record + `benchmarks/independent/plane-increment-2.json` next to the first. + - _Requirements: 6.2, 6.4_ + - [ ] 9. Handover - `ops/README.md`: how to run a Run locally and what an ax deployment would need (Kubernetes, Agent Substrate, the runner image, the provider extension); memory note updated; scripts that the tasks replaced diff --git a/plane/runs/validation-46.yaml b/plane/runs/validation-46.yaml index 8fe8800..93ff97a 100644 --- a/plane/runs/validation-46.yaml +++ b/plane/runs/validation-46.yaml @@ -1,5 +1,5 @@ # generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set ~/ousast-results/independent-v2-sets/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --runner-image ~/.cache/ousast/ax-src/runner-image`; do not edit, regenerate. -# 15 cases, 46 candidates (43 kept by the recorded triage); usd ceiling 2.02. +# 15 cases, 46 candidates (43 kept by the recorded triage); usd ceiling 3.03. apiVersion: openultrasast.io/v1alpha1 kind: Run metadata: @@ -561,3 +561,378 @@ spec: - agreed.json - disputed.json budget: *id015 + - name: budibase-mongo-template-nosqli-vc + task: verify-c-budibase-mongo-template-nosqli + dependsOn: + - budibase-mongo-template-nosqli-agree + inputs: + facts: budibase-mongo-template-nosqli-facts/facts.json + only: budibase-mongo-template-nosqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.06 + calls: 16 + - name: budibase-mongo-template-nosqli-final + task: agree-budibase-mongo-template-nosqli + inputs: + pass_a: budibase-mongo-template-nosqli-va/units.jsonl + pass_b: budibase-mongo-template-nosqli-vb/units.jsonl + pass_c: budibase-mongo-template-nosqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: cct-studio-server-cmdi-vc + task: verify-c-cct-studio-server-cmdi + dependsOn: + - cct-studio-server-cmdi-agree + inputs: + facts: cct-studio-server-cmdi-facts/facts.json + only: cct-studio-server-cmdi-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.04 + calls: 8 + - name: cct-studio-server-cmdi-final + task: agree-cct-studio-server-cmdi + inputs: + pass_a: cct-studio-server-cmdi-va/units.jsonl + pass_b: cct-studio-server-cmdi-vb/units.jsonl + pass_c: cct-studio-server-cmdi-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: contextforge-prompt-ssti-vc + task: verify-c-contextforge-prompt-ssti + dependsOn: + - contextforge-prompt-ssti-agree + inputs: + facts: contextforge-prompt-ssti-facts/facts.json + only: contextforge-prompt-ssti-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.04 + calls: 8 + - name: contextforge-prompt-ssti-final + task: agree-contextforge-prompt-ssti + inputs: + pass_a: contextforge-prompt-ssti-va/units.jsonl + pass_b: contextforge-prompt-ssti-vb/units.jsonl + pass_c: contextforge-prompt-ssti-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: froxlor-admin-ip-second-order-sqli-vc + task: verify-c-froxlor-admin-ip-second-order-sqli + dependsOn: + - froxlor-admin-ip-second-order-sqli-agree + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + only: froxlor-admin-ip-second-order-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-final + task: agree-froxlor-admin-ip-second-order-sqli + inputs: + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: fuxa-tdengine-escape-sqli-vc + task: verify-c-fuxa-tdengine-escape-sqli + dependsOn: + - fuxa-tdengine-escape-sqli-agree + inputs: + facts: fuxa-tdengine-escape-sqli-facts/facts.json + only: fuxa-tdengine-escape-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.12 + calls: 8 + - name: fuxa-tdengine-escape-sqli-final + task: agree-fuxa-tdengine-escape-sqli + inputs: + pass_a: fuxa-tdengine-escape-sqli-va/units.jsonl + pass_b: fuxa-tdengine-escape-sqli-vb/units.jsonl + pass_c: fuxa-tdengine-escape-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: lmdeploy-media-url-ssrf-vc + task: verify-c-lmdeploy-media-url-ssrf + dependsOn: + - lmdeploy-media-url-ssrf-agree + inputs: + facts: lmdeploy-media-url-ssrf-facts/facts.json + only: lmdeploy-media-url-ssrf-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: lmdeploy-media-url-ssrf-final + task: agree-lmdeploy-media-url-ssrf + inputs: + pass_a: lmdeploy-media-url-ssrf-va/units.jsonl + pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl + pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: medplum-external-auth-open-redirect-vc + task: verify-c-medplum-external-auth-open-redirect + dependsOn: + - medplum-external-auth-open-redirect-agree + inputs: + facts: medplum-external-auth-open-redirect-facts/facts.json + only: medplum-external-auth-open-redirect-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: medplum-external-auth-open-redirect-final + task: agree-medplum-external-auth-open-redirect + inputs: + pass_a: medplum-external-auth-open-redirect-va/units.jsonl + pass_b: medplum-external-auth-open-redirect-vb/units.jsonl + pass_c: medplum-external-auth-open-redirect-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: pgadmin-maintenance-sqli-vc + task: verify-c-pgadmin-maintenance-sqli + dependsOn: + - pgadmin-maintenance-sqli-agree + inputs: + facts: pgadmin-maintenance-sqli-facts/facts.json + only: pgadmin-maintenance-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.45 + calls: 96 + - name: pgadmin-maintenance-sqli-final + task: agree-pgadmin-maintenance-sqli + inputs: + pass_a: pgadmin-maintenance-sqli-va/units.jsonl + pass_b: pgadmin-maintenance-sqli-vb/units.jsonl + pass_c: pgadmin-maintenance-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: qwed-sympy-eval-vc + task: verify-c-qwed-sympy-eval + dependsOn: + - qwed-sympy-eval-agree + inputs: + facts: qwed-sympy-eval-facts/facts.json + only: qwed-sympy-eval-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: qwed-sympy-eval-final + task: agree-qwed-sympy-eval + inputs: + pass_a: qwed-sympy-eval-va/units.jsonl + pass_b: qwed-sympy-eval-vb/units.jsonl + pass_c: qwed-sympy-eval-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: reactpress-findall-column-sqli-vc + task: verify-c-reactpress-findall-column-sqli + dependsOn: + - reactpress-findall-column-sqli-agree + inputs: + facts: reactpress-findall-column-sqli-facts/facts.json + only: reactpress-findall-column-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: reactpress-findall-column-sqli-final + task: agree-reactpress-findall-column-sqli + inputs: + pass_a: reactpress-findall-column-sqli-va/units.jsonl + pass_b: reactpress-findall-column-sqli-vb/units.jsonl + pass_c: reactpress-findall-column-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: unleash-addon-webhook-ssrf-vc + task: verify-c-unleash-addon-webhook-ssrf + dependsOn: + - unleash-addon-webhook-ssrf-agree + inputs: + facts: unleash-addon-webhook-ssrf-facts/facts.json + only: unleash-addon-webhook-ssrf-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.03 + calls: 8 + - name: unleash-addon-webhook-ssrf-final + task: agree-unleash-addon-webhook-ssrf + inputs: + pass_a: unleash-addon-webhook-ssrf-va/units.jsonl + pass_b: unleash-addon-webhook-ssrf-vb/units.jsonl + pass_c: unleash-addon-webhook-ssrf-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: winml-cli-serve-cors-vc + task: verify-c-winml-cli-serve-cors + dependsOn: + - winml-cli-serve-cors-agree + inputs: + facts: winml-cli-serve-cors-facts/facts.json + only: winml-cli-serve-cors-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: winml-cli-serve-cors-final + task: agree-winml-cli-serve-cors + inputs: + pass_a: winml-cli-serve-cors-va/units.jsonl + pass_b: winml-cli-serve-cors-vb/units.jsonl + pass_c: winml-cli-serve-cors-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: wp-directorist-avatar-ssrf-vc + task: verify-c-wp-directorist-avatar-ssrf + dependsOn: + - wp-directorist-avatar-ssrf-agree + inputs: + facts: wp-directorist-avatar-ssrf-facts/facts.json + only: wp-directorist-avatar-ssrf-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.03 + calls: 8 + - name: wp-directorist-avatar-ssrf-final + task: agree-wp-directorist-avatar-ssrf + inputs: + pass_a: wp-directorist-avatar-ssrf-va/units.jsonl + pass_b: wp-directorist-avatar-ssrf-vb/units.jsonl + pass_c: wp-directorist-avatar-ssrf-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: wp-gopay-log-filter-sqli-vc + task: verify-c-wp-gopay-log-filter-sqli + dependsOn: + - wp-gopay-log-filter-sqli-agree + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + only: wp-gopay-log-filter-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-final + task: agree-wp-gopay-log-filter-sqli + inputs: + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: wp-groundhogg-confirm-redirect-vc + task: verify-c-wp-groundhogg-confirm-redirect + dependsOn: + - wp-groundhogg-confirm-redirect-agree + inputs: + facts: wp-groundhogg-confirm-redirect-facts/facts.json + only: wp-groundhogg-confirm-redirect-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.05 + calls: 8 + - name: wp-groundhogg-confirm-redirect-final + task: agree-wp-groundhogg-confirm-redirect + inputs: + pass_a: wp-groundhogg-confirm-redirect-va/units.jsonl + pass_b: wp-groundhogg-confirm-redirect-vb/units.jsonl + pass_c: wp-groundhogg-confirm-redirect-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 diff --git a/plane/tasks/validation-46.yaml b/plane/tasks/validation-46.yaml index 6008036..c139e60 100644 --- a/plane/tasks/validation-46.yaml +++ b/plane/tasks/validation-46.yaml @@ -1738,3 +1738,468 @@ spec: workspaces: - name: wp-groundhogg-confirm-redirect-inputs path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-budibase-mongo-template-nosqli + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: budibase-mongo-template-nosqli-vulnerable + path: /workspace/case + - name: budibase-mongo-template-nosqli-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-cct-studio-server-cmdi + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: cct-studio-server-cmdi-vulnerable + path: /workspace/case + - name: cct-studio-server-cmdi-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-contextforge-prompt-ssti + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: contextforge-prompt-ssti-vulnerable + path: /workspace/case + - name: contextforge-prompt-ssti-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-froxlor-admin-ip-second-order-sqli + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: froxlor-admin-ip-second-order-sqli-vulnerable + path: /workspace/case + - name: froxlor-admin-ip-second-order-sqli-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-fuxa-tdengine-escape-sqli + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: fuxa-tdengine-escape-sqli-vulnerable + path: /workspace/case + - name: fuxa-tdengine-escape-sqli-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-lmdeploy-media-url-ssrf + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: lmdeploy-media-url-ssrf-vulnerable + path: /workspace/case + - name: lmdeploy-media-url-ssrf-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-medplum-external-auth-open-redirect + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: medplum-external-auth-open-redirect-vulnerable + path: /workspace/case + - name: medplum-external-auth-open-redirect-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-pgadmin-maintenance-sqli + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: pgadmin-maintenance-sqli-vulnerable + path: /workspace/case + - name: pgadmin-maintenance-sqli-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-qwed-sympy-eval + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: qwed-sympy-eval-vulnerable + path: /workspace/case + - name: qwed-sympy-eval-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-reactpress-findall-column-sqli + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: reactpress-findall-column-sqli-vulnerable + path: /workspace/case + - name: reactpress-findall-column-sqli-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-unleash-addon-webhook-ssrf + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: unleash-addon-webhook-ssrf-vulnerable + path: /workspace/case + - name: unleash-addon-webhook-ssrf-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-winml-cli-serve-cors + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: winml-cli-serve-cors-vulnerable + path: /workspace/case + - name: winml-cli-serve-cors-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-wp-directorist-avatar-ssrf + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: wp-directorist-avatar-ssrf-vulnerable + path: /workspace/case + - name: wp-directorist-avatar-ssrf-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-wp-gopay-log-filter-sqli + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: wp-gopay-log-filter-sqli-vulnerable + path: /workspace/case + - name: wp-gopay-log-filter-sqli-inputs + path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: verify-c-wp-groundhogg-confirm-redirect + atespace: default + annotations: + openultrasast.io/model: deepseek-flash +spec: + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + command: + - verify + env: + - name: OUSAST_WORKSPACE_DIR + value: /workspace/case/repo + - name: OUSAST_INPUT_CANDIDATES + value: /workspace/inputs/candidates.json + - name: OUSAST_PASS + value: c + resources: + requests: + cpu: 250m + memory: 384Mi + limits: + cpu: '1' + memory: 1Gi + workspaces: + - name: wp-groundhogg-confirm-redirect-vulnerable + path: /workspace/case + - name: wp-groundhogg-confirm-redirect-inputs + path: /workspace/inputs diff --git a/src/openultrasast/plane/generate.py b/src/openultrasast/plane/generate.py index 409af53..517b08d 100644 --- a/src/openultrasast/plane/generate.py +++ b/src/openultrasast/plane/generate.py @@ -233,12 +233,31 @@ def verify_budget(item: CaseInputs) -> dict[str, float | int]: return {"usd": math.ceil(HEADROOM * share * 100 - 1e-9) / 100, "calls": max(hunts, 1) * CALLS_PER_HUNT} +def _tiebreak_entries(item: CaseInputs) -> list[dict[str, Any]]: + """The tie-break (Requirement 6.4): verify pass c on the candidates a and b disputed, then agree 2-of-3. Fresh + budget dicts, so YAML adds no anchor to the entries before them.""" + case_id = item.case["id"] + return [ + { + "name": f"{case_id}-vc", "task": f"verify-c-{case_id}", "dependsOn": [f"{case_id}-agree"], + "inputs": {"facts": f"{case_id}-facts/facts.json", "only": f"{case_id}-agree/disputed.json"}, + "outputs": ["units.jsonl", "summary.json"], "budget": verify_budget(item), + }, + { + "name": f"{case_id}-final", "task": f"agree-{case_id}", + "inputs": {pass_: f"{case_id}-v{pass_[-1]}/units.jsonl" for pass_ in ("pass_a", "pass_b", "pass_c")}, + "outputs": ["agreed.json", "disputed.json"], "budget": {"usd": 0, "calls": 0}, + }, + ] # fmt: skip + + def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str) -> dict[str, str]: """Relative path under the plane root -> file text, for every generated manifest.""" header = f"# generated by `{command}`; do not edit, regenerate.\n" out: dict[str, str] = {} tasks: list[dict[str, Any]] = [] entries: list[dict[str, Any]] = [] + tiebreak: list[dict[str, Any]] = [] for item in cases: case_id = item.case["id"] pinned, inputs = _workspaces(item) @@ -270,6 +289,11 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: "inputs": {"pass_a": f"{case_id}-va/units.jsonl", "pass_b": f"{case_id}-vb/units.jsonl"}, "outputs": ["agreed.json", "disputed.json"], "budget": free, }) # fmt: skip + env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/candidates.json", "OUSAST_PASS": "c"} + tasks.append(_task(templates["verify"], f"verify-c-{case_id}", env, repo)) + tiebreak.extend(_tiebreak_entries(item)) + tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first + entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state out[f"tasks/{run_name}.yaml"] = _dump(header, tasks) run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": {"name": run_name}, "spec": {"tasks": entries}} total = sum(float(e["budget"]["usd"]) for e in entries) diff --git a/src/openultrasast/plane/tasks/agree.py b/src/openultrasast/plane/tasks/agree.py index ef0ce3d..c1d8c0e 100644 --- a/src/openultrasast/plane/tasks/agree.py +++ b/src/openultrasast/plane/tasks/agree.py @@ -17,7 +17,13 @@ divide by the candidates before triage when `case.json` gives that count, as the Env contract: `OUSAST_INPUT_PASS_A`, `OUSAST_INPUT_PASS_B` (a verify output directory or its `units.jsonl`), `OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (the generator's `case.json`: `family`, `sites`, `ranges`, `sites_in_set`, `cost`; or a `population-v2.toml`-shaped TOML with `OUSAST_CASE_ID` -naming the case). Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished. +naming the case), optional `OUSAST_INPUT_PASS_C` (the tie-break pass: `verify` restricted to the candidates a and +b disputed). Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished. + +With pass c the decision is 2 of 3 for every candidate pass c asked about and the a/b rule for the rest, so pass c +alone never decides; `metrics.tiebreak` counts the candidates asked and the decisions that flipped against a/b, +and names the rule. The declared-site metrics and `agreed`/`disputed` are on the final decision; `usd_total`, +`hunts` and the per-hunt means include pass c. Without pass c every number is the two-pass one. """ from __future__ import annotations @@ -122,6 +128,28 @@ def _per_candidate(total: float | None, count: int) -> float | None: return None if total is None or not count else round(total / count, 6) +RULE_A_B = "agreed when passes a and b both flagged the candidate (no pass c given)" +RULE_2_OF_3 = ( + "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in " + "pass c is agreed when passes a and b both flagged it (pass c alone never decides)" +) + + +def _votes(candidate: str, flagged: Sequence[Mapping[str, Any]]) -> int: + return sum(candidate in f for f in flagged) + + +def _pick(candidate: str, flagged: Sequence[Mapping[str, Mapping[str, Any]]]) -> dict[str, Any]: + return dict(next(f[candidate] for f in flagged if candidate in f)) + + +def _decided(candidate: str, asked_c: set[str], flagged: Sequence[Mapping[str, Any]]) -> bool: + """The final decision: 2 of 3 when pass c asked the candidate, else both of a and b.""" + if candidate in asked_c: + return _votes(candidate, flagged) >= 2 + return candidate in flagged[0] and candidate in flagged[1] + + def run( output_dir: Path, candidates: object, @@ -129,15 +157,28 @@ def run( pass_b: Sequence[Mapping[str, Any]], *, sites: Mapping[str, Any] | None = None, + pass_c: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any]: output_dir.mkdir(parents=True, exist_ok=True) family, unique = load_candidates(candidates) names = [f"{p}::{fn}" for p, fn, _ in unique] lines = {f"{p}::{fn}": ln for p, fn, ln in unique} - asked = [{f"{r['path']}::{c[1]}" for r in rows for c in r.get("candidates", [])} for rows in (pass_a, pass_b)] - flagged = [{str(f["candidate"]): dict(f) for r in rows for f in r.get("flagged", [])} for rows in (pass_a, pass_b)] - agreed, disputed = agreement([list(flagged[0].values()), list(flagged[1].values())]) - usd_by, turns_by = _per_candidate_cost([*pass_a, *pass_b]) + passes = (pass_a, pass_b, pass_c or []) + asked = [{f"{r['path']}::{c[1]}" for r in rows for c in r.get("candidates", [])} for rows in passes] + flagged = [{str(f["candidate"]): dict(f) for r in rows for f in r.get("flagged", [])} for rows in passes] + ab_agreed, _ = agreement([list(flagged[0].values()), list(flagged[1].values())]) + ab = {str(f["candidate"]) for f in ab_agreed} + final = {c for c in set(flagged[0]) | set(flagged[1]) | set(flagged[2]) if _decided(c, asked[2], flagged)} + agreed = [dict(f, passes=_votes(f["candidate"], flagged)) for f in ab_agreed if f["candidate"] in final] + agreed += [dict(_pick(c, flagged), passes=_votes(c, flagged)) for c in sorted(final - ab)] + agreed.sort(key=lambda f: str(f["candidate"])) + some = (set(flagged[0]) | set(flagged[1]) | set(flagged[2])) - final + disputed = [dict(_pick(c, flagged), passes=_votes(c, flagged)) for c in sorted(some)] + tiebreak = { + "asked": len(asked[2]), "flipped_to_agreed": len(final - ab), "flipped_to_rejected": len(ab - final), + "rule": RULE_2_OF_3 if pass_c is not None else RULE_A_B, + } # fmt: skip + usd_by, turns_by = _per_candidate_cost([*pass_a, *pass_b, *(pass_c or [])]) case = dict(sites) if sites else None if case is not None and "family" not in case and family: case["family"] = family @@ -147,19 +188,20 @@ def run( rows: list[dict[str, Any]] = [] for name in names: in_a, in_b = (name in flagged[0] if name in asked[0] else None), (name in flagged[1] if name in asked[1] else None) - finding = flagged[0].get(name) or flagged[1].get(name) + in_c = name in flagged[2] if name in asked[2] else None + finding = flagged[0].get(name) or flagged[1].get(name) or flagged[2].get(name) site = anchor_site(name, finding["site"]) if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}" site_match = None if assessed and case is not None and isinstance(ranges, Mapping): site_match = matches_declared({"site": site, "family": (finding or {}).get("family", family)}, case, ranges) cost = usd_by.get(name) rows.append({ - "candidate": name, "site": site, "a": in_a, "b": in_b, "agreed": bool(in_a and in_b), "disputed": bool(in_a) != bool(in_b), + "candidate": name, "site": site, "a": in_a, "b": in_b, "c": in_c, "agreed": name in final, "disputed": bool(in_a) != bool(in_b), "site_match": site_match, "usd": (None if cost is None else round(cost, 6)), "turns": turns_by.get(name, 0), "finding": finding, }) # fmt: skip unasked = [r["candidate"] for r in rows if r["a"] is None or r["b"] is None] declared = [r for r in rows if r["site_match"]] - hunts = [*pass_a, *pass_b] + hunts = [*pass_a, *pass_b, *(pass_c or [])] usd_total = None if any(r.get("usd") is None for r in hunts) else round(sum(float(r["usd"]) for r in hunts), 6) reference: Mapping[str, Any] = (case or {}).get("cost") or {} before = int(reference.get("candidates_before_triage") or len(names)) @@ -177,6 +219,7 @@ def run( "usd_total_with_recorded_triage": with_triage, "cost_per_candidate_with_recorded_triage": _per_candidate(with_triage, before), "turns_per_hunt": _mean([float(r.get("turns") or 0) for r in hunts]), "calls_per_hunt": _mean([float((r.get("usage") or {}).get("calls") or 0) for r in hunts]), + "tiebreak": tiebreak, } # fmt: skip (output_dir / "agreed.json").write_text( json.dumps({"family": family, "candidates": rows, "agreed": agreed, "disputed": disputed, "metrics": metrics}, indent=1) + "\n" @@ -201,7 +244,8 @@ def main() -> int: candidates = json.loads(Path(os.environ["OUSAST_INPUT_CANDIDATES"]).read_text()) sites_path = Path(os.environ["OUSAST_INPUT_SITES"]) if os.environ.get("OUSAST_INPUT_SITES") else None sites = load_sites(sites_path, os.environ.get("OUSAST_CASE_ID")) - summary = run(output_dir, candidates, pass_a, pass_b, sites=sites) + pass_c = load_pass(Path(os.environ["OUSAST_INPUT_PASS_C"])) if os.environ.get("OUSAST_INPUT_PASS_C") else None + summary = run(output_dir, candidates, pass_a, pass_b, sites=sites, pass_c=pass_c) except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback summary = { "status": "failed", "units_done": 0, "units_total": 0, "usd": 0, "calls": 0, "usage": dict.fromkeys(USAGE_FIELDS, 0), @@ -214,7 +258,18 @@ def main() -> int: return EXIT_CODES.get(str(summary["status"]), 2) -__all__ = ["LINE_WINDOW", "agreement", "anchor_site", "load_pass", "load_sites", "main", "matches_declared", "run"] +__all__ = [ + "LINE_WINDOW", + "RULE_2_OF_3", + "RULE_A_B", + "agreement", + "anchor_site", + "load_pass", + "load_sites", + "main", + "matches_declared", + "run", +] if __name__ == "__main__": diff --git a/src/openultrasast/plane/tasks/verify.py b/src/openultrasast/plane/tasks/verify.py index 494f9b8..67bc5c8 100644 --- a/src/openultrasast/plane/tasks/verify.py +++ b/src/openultrasast/plane/tasks/verify.py @@ -10,8 +10,10 @@ completes, read back on start: Requirement 2.2). The chat client is wrapped by ` last -- names the bound Model (Requirement 7.2) and is `done` only when every unit finished. Env contract: `OUSAST_WORKSPACE_DIR`, `OUSAST_OUTPUT_DIR`, `OUSAST_INPUT_CANDIDATES`, `OUSAST_INPUT_FACTS` -(optional), `OUSAST_BUDGET_USD`, `OUSAST_BUDGET_CALLS`, `OUSAST_MODEL`, `OUSAST_MODEL_PARAMS`, `OUSAST_PASS`. -Exit 0 done, 2 failed, 3 unfinished. +(optional), `OUSAST_BUDGET_USD`, `OUSAST_BUDGET_CALLS`, `OUSAST_MODEL`, `OUSAST_MODEL_PARAMS`, `OUSAST_PASS`, +`OUSAST_INPUT_ONLY` (optional: a JSON list of `path::function` names, `[path, function, ...]` rows or finding +objects with a `candidate` -- `agree`'s `disputed.json` -- restricting the hunt to those candidates; the tie-break +pass c. An empty list is a `done` task with 0 units and no model call). Exit 0 done, 2 failed, 3 unfinished. """ from __future__ import annotations @@ -95,6 +97,34 @@ def load_candidates(payload: object) -> tuple[str | None, list[Candidate]]: return family, list(unique.values()) +def only_names(payload: object) -> set[str]: + """The `path::function` names of an `OUSAST_INPUT_ONLY` list (names, rows, or `agree`'s finding objects).""" + rows = payload.get("disputed", []) if isinstance(payload, Mapping) else payload + if not isinstance(rows, list): + raise ValueError("OUSAST_INPUT_ONLY must be a JSON list (e.g. agree's disputed.json)") + names = set() + for row in rows: + if isinstance(row, Mapping): + names.add(str(row["candidate"])) + elif isinstance(row, list | tuple): + names.add(f"{row[0]}::{row[1]}") + else: + names.add(str(row)) + return names + + +def restrict(unique: Sequence[Candidate], only: set[str] | None) -> list[Candidate]: + """The candidates named by `only` (all of them when `only` is None).""" + return list(unique) if only is None else [c for c in unique if f"{c[0]}::{c[1]}" in only] + + +class _NoModel: + """The client of a task with nothing to ask: any call is a defect, never a silent spend.""" + + def complete(self, **kw: Any) -> ChatResponse: + raise RuntimeError("no model call is expected: the candidate restriction is empty") + + def callers_of(facts: Mapping[str, Any] | None, path: str, function: str) -> list[dict[str, Any]]: """Known call sites of `path::function` from `facts.json`: the task-5 shape (`callers` keyed by `path::function`, rows `{"path", "line", "enclosing"}`) or the design's per-file shape (`{path: {"callers": @@ -224,10 +254,12 @@ def run( model: str = "", pass_label: str = "a", family: str | None = None, + only: set[str] | None = None, ) -> dict[str, Any]: """Hunt every unfinished unit, append `units.jsonl` per hunt, write `summary.json` last; returns the summary. - `budget` is the meter to run under (built from `client` when absent); `client` is what the meter wraps. + `budget` is the meter to run under (built from `client` when absent); `client` is what the meter wraps; + `only` restricts the hunt to those `path::function` candidates. """ output_dir.mkdir(parents=True, exist_ok=True) log, summary_path = output_dir / "units.jsonl", output_dir / "summary.json" @@ -235,7 +267,7 @@ def run( model = model or getattr(metered, "model", "") or "unbound" recorded_family, unique = load_candidates(candidates) family = family or recorded_family - units = units_of(unique) + units = units_of(restrict(unique, only)) rows = read_units(log) done = {unit_key(r["path"], r["candidates"]): r for r in rows if "error" not in r} status, reason = "done", "" @@ -278,6 +310,8 @@ def run( "status": status, "units_done": len(done), "units_total": len(units), "usd": usd, "calls": calls, "usage": usage, "priced": metered.priced, "model": model, "pass": pass_label, "family": family, } # fmt: skip + if only is not None: + summary["only"] = len(restrict(unique, only)) if reason: summary["reason"] = reason summary_path.write_text(json.dumps(summary, indent=1) + "\n") @@ -336,13 +370,16 @@ def main() -> int: params = json.loads(os.environ.get("OUSAST_MODEL_PARAMS", "") or "{}") candidates = json.loads(candidates_path.read_text()) facts = json.loads(facts_path.read_text()) if facts_path is not None else None - client = build_client() + only_path = _env_path("OUSAST_INPUT_ONLY", required=False) + only = only_names(json.loads(only_path.read_text())) if only_path is not None else None + nothing = only is not None and not restrict(load_candidates(candidates)[1], only) + client: ChatClient = _NoModel() if nothing else build_client() metered = MeteredClient( client, prices=params, budget_usd=_env_number("OUSAST_BUDGET_USD", float), budget_calls=_env_number("OUSAST_BUDGET_CALLS", int) ) summary = run( workspace, output_dir, candidates, facts, client=client, budget=metered, model=model, pass_label=pass_label, - family=os.environ.get("OUSAST_FAMILY") or None, + family=os.environ.get("OUSAST_FAMILY") or None, only=only, ) # fmt: skip except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback, never a silent zero summary = _failed(output_dir, traceback.format_exc()[-2000:], model, pass_label) @@ -364,6 +401,8 @@ __all__ = [ "hunt_prompt", "load_candidates", "main", + "only_names", + "restrict", "run", "units_of", ] diff --git a/tests/test_plane_increment.py b/tests/test_plane_increment.py index 8fbfaec..96df2a6 100644 --- a/tests/test_plane_increment.py +++ b/tests/test_plane_increment.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import re import shutil @@ -53,7 +54,7 @@ def test_model_prices_are_the_endpoint_prices() -> None: def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None: run, manifests = reconciler.load_run(RUN) - assert len(run.tasks) == 60 + assert len(run.tasks) == 90 population = {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]} ax_names: set[str] = set() for task in manifests.tasks.values(): @@ -69,7 +70,7 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None: for name, sha in workspace.pins.items(): case_id = binding.name.removesuffix("-vulnerable") assert (name, sha) == ("repo", population[case_id]["vulnerable"]) - if entry.name.endswith(("-va", "-vb")): + if entry.name.endswith(("-va", "-vb", "-vc")): assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls env = {e.name: e.value for e in task.env} assert env["OUSAST_PASS"] == entry.name[-1] @@ -83,6 +84,34 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None: ax_names.add(docs[-1]["metadata"]["name"]) +# sha256 of the Run's entries as the first measurement ran them (plane-increment-1.json): the text after +# `` tasks:`` of plane/runs/validation-46.yaml at c5e6ed3. A rerun keeps their ``done`` state only while they match. +FIRST_RUN_ENTRIES = "412ad6171cf096d5d7f6b8e7358374077e2beabf5f4dff35a2801d37c341b97e" + + +def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical() -> None: + text = RUN.read_text().split(" tasks:\n", 1)[1] + first = text.split(" - name: budibase-mongo-template-nosqli-vc\n", 1)[0] + assert hashlib.sha256(first.encode()).hexdigest() == FIRST_RUN_ENTRIES + run, manifests = reconciler.load_run(RUN) + old, new = run.tasks[:60], run.tasks[60:] + assert [e.name.rsplit("-", 1)[1] for e in old] == ["facts", "va", "vb", "agree"] * 15 + assert [e.name.rsplit("-", 1)[1] for e in new] == ["vc", "final"] * 15 + for vc, final in zip(new[::2], new[1::2], strict=True): + case = vc.name.removesuffix("-vc") + agree_entry = run.task(f"{case}-agree") + assert vc.inputs == {"facts": f"{case}-facts/facts.json", "only": f"{case}-agree/disputed.json"} + assert vc.depends_on == (f"{case}-agree",) and vc.budget == run.task(f"{case}-va").budget + env = {e.name: e.value for e in manifests.tasks[vc.task].env} + assert env["OUSAST_PASS"] == "c" and env["OUSAST_INPUT_CANDIDATES"] == "/workspace/inputs/candidates.json" + assert final.task == agree_entry.task, "the final agree binds the same static inputs as the first" + assert final.inputs == {**agree_entry.inputs, "pass_c": f"{case}-vc/units.jsonl"} + assert set(final.producers) == {f"{case}-va", f"{case}-vb", f"{case}-vc"} + for entry in (vc, final): + name = reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT) + assert len(name) <= reconciler.TASK_NAME_LIMIT + + def test_inputs_carry_the_validation_set() -> None: manifests = load_manifests(sorted((PLANE / "workspaces").glob("*-inputs.yaml"))) assert len(manifests.workspaces) == 15 diff --git a/tests/test_plane_reconciler.py b/tests/test_plane_reconciler.py index dd0f531..70a06cf 100644 --- a/tests/test_plane_reconciler.py +++ b/tests/test_plane_reconciler.py @@ -448,6 +448,30 @@ def test_unfinished_stops_dependants_and_rerun_skips_done(fake: Fake, tmp_path: assert statuses("resume") == {"repo-facts": "done", "verify": "done", "agree": "done"} +TIEBREAK = """ + - name: tiebreak + task: verify + dependsOn: [agree] + inputs: {only: verify/agreed.json, facts: repo-facts/facts.json} + outputs: [agreed.json] + budget: {usd: 1, calls: 8} +""" + + +def test_a_rerun_of_a_run_that_gained_tasks_runs_only_the_new_ones(fake: Fake, tmp_path: Path) -> None: + manifest = write_run(tmp_path, "grow") + assert reconciler.run(manifest, ax=str(fake.ax)) == "done" + first = state_of("grow")["tasks"] + (fake.root / "ax.log").unlink() + fake.script({**SCRIPT, "tiebreak": SCRIPT["verify"]}) + write_run(tmp_path, "grow", CHAIN + TIEBREAK) # the same Run, one task appended (the tie-break pass) + assert reconciler.run(manifest, ax=str(fake.ax)) == "done" + assert [name for _, name in fake.events("apply")] == ["tiebreak"], "done tasks keep their state; only the new task runs" + assert statuses("grow") == {"repo-facts": "done", "verify": "done", "agree": "done", "tiebreak": "done"} + assert {k: v for k, v in state_of("grow")["tasks"].items() if k != "tiebreak"} == first + assert sorted(n for _, n in fake.events("fetched")) == ["tiebreak FACTS 200", "tiebreak ONLY 200"], "inputs of done producers" + + def test_failed_task_halts_the_run(fake: Fake, tmp_path: Path) -> None: script = json.loads(json.dumps(SCRIPT)) failed = {"status": "failed", "units_done": 0, "units_total": 3, "reason": "402 insufficient balance"} diff --git a/tests/test_plane_tiebreak.py b/tests/test_plane_tiebreak.py new file mode 100644 index 0000000..61a79ed --- /dev/null +++ b/tests/test_plane_tiebreak.py @@ -0,0 +1,150 @@ +"""The tie-break of the first increment's revisit (ai-service-plane Requirement 6.4, task 8.1): `verify` restricted +to the candidates passes a and b disputed (`OUSAST_INPUT_ONLY`), and `agree` deciding 2-of-3 where pass c asked.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from test_plane_verify import CANDIDATES, FACTS, FLASH, HuntClient, _two_passes, _workspace + +from openultrasast.plane.budget import MeteredClient +from openultrasast.plane.tasks import agree, verify +from openultrasast.tool_hunter import CLIENT_ENV + +SITES = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"], "ranges": {"app.py": [[40, 41]]}} + + +def _verify(root: Path, out: Path, client: HuntClient, only: set[str] | None, label: str = "c") -> dict: + metered = MeteredClient(client, prices=FLASH) # type: ignore[arg-type] + return verify.run(root, out, CANDIDATES, FACTS, client=client, budget=metered, model="deepseek-chat", pass_label=label, only=only) + + +def test_only_restricts_the_hunt_to_the_named_candidates(tmp_path: Path) -> None: + client = HuntClient({"app.py::other"}) + summary = _verify(_workspace(tmp_path), tmp_path / "c", client, {"app.py::other", "gone.py::f"}) + assert summary["status"] == "done" and summary["units_total"] == 1 and summary["only"] == 1 and summary["pass"] == "c" + prompts = client.prompts() + assert prompts and all("`other`" in p and "`run`" not in p and "db.py" not in p for p in prompts) + rows = [json.loads(line) for line in (tmp_path / "c" / "units.jsonl").read_text().splitlines()] + assert [r["candidates"] for r in rows] == [[["app.py", "other", 5]]] + assert [f["candidate"] for f in rows[0]["flagged"]] == ["app.py::other"] + + +def test_only_names_reads_disputed_json_rows_and_names() -> None: + disputed = [{"candidate": "a.py::f", "site": "a.py:1:f", "passes": 1}] + assert verify.only_names(disputed) == {"a.py::f"} + assert verify.only_names({"disputed": disputed}) == {"a.py::f"} + assert verify.only_names([["b.py", "g", 3], "c.py::h"]) == {"b.py::g", "c.py::h"} + with pytest.raises(ValueError): + verify.only_names("a.py::f") + + +def test_an_empty_restriction_is_done_with_zero_units_and_no_call(tmp_path: Path) -> None: + client = HuntClient({"app.py::run"}) + summary = _verify(_workspace(tmp_path), tmp_path / "c", client, set()) + assert summary["status"] == "done" and (summary["units_done"], summary["units_total"], summary["calls"]) == (0, 0, 0) + assert client.calls == [] and (tmp_path / "c" / "units.jsonl").read_text() == "" + + +def test_main_with_an_empty_only_list_needs_no_model_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root, out = _workspace(tmp_path), tmp_path / "out" + (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES)) + (tmp_path / "disputed.json").write_text("[]\n") + for name in ("DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", CLIENT_ENV, "OUSAST_CLIENT"): + monkeypatch.delenv(name, raising=False) + env = { + "OUSAST_WORKSPACE_DIR": str(root), "OUSAST_OUTPUT_DIR": str(out), "OUSAST_INPUT_CANDIDATES": str(tmp_path / "candidates.json"), + "OUSAST_INPUT_ONLY": str(tmp_path / "disputed.json"), "OUSAST_MODEL": "deepseek-chat", "OUSAST_PASS": "c", + } # fmt: skip + for key, value in env.items(): + monkeypatch.setenv(key, value) + assert verify.main() == 0 + summary = json.loads((out / "summary.json").read_text()) + assert (summary["status"], summary["units_total"], summary["calls"], summary["only"]) == ("done", 0, 0, 0) + + +def _pass_c(tmp_path: Path, root: Path, disputed: Path, flag: set[str]) -> list[dict]: + only = verify.only_names(json.loads(disputed.read_text())) + _verify(root, tmp_path / "pass-c", HuntClient(flag), only) + return agree.load_pass(tmp_path / "pass-c") + + +def _final(tmp_path: Path, a: Path, b: Path, c: list[dict] | None) -> dict: + return agree.run(tmp_path / "final", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES, pass_c=c) + + +def test_a_and_c_agreeing_flips_a_disputed_candidate_to_agreed(tmp_path: Path) -> None: + root, a, b = _two_passes(tmp_path) # a: run, query; b: run, other -> disputed: other (b), query (a) + first = agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES) + c = _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"db.py::query", "app.py::run"}) + summary = _final(tmp_path, a, b, c) + report = json.loads((tmp_path / "final" / "agreed.json").read_text()) + rows = {r["candidate"]: (r["a"], r["b"], r["c"], r["agreed"]) for r in report["candidates"]} + assert rows == { + "app.py::run": (True, True, None, True), # not asked in c: the a/b rule + "app.py::other": (False, True, False, False), # b alone + "db.py::query": (True, False, True, True), # a + c + } + assert [f["candidate"] for f in report["agreed"]] == ["app.py::run", "db.py::query"] + assert [f["candidate"] for f in report["disputed"]] == ["app.py::other"] + metrics = summary["metrics"] + assert metrics["tiebreak"]["asked"] == 2 and metrics["tiebreak"]["flipped_to_agreed"] == 1 + assert metrics["tiebreak"]["flipped_to_rejected"] == 0 and "2 of passes a, b, c" in metrics["tiebreak"]["rule"] + assert metrics["agreed"] == 2 and metrics["disputed"] == 1 + assert first["metrics"]["declared_sites_matched_agreed"] == 1 and metrics["declared_sites_matched_agreed"] == 2 + assert metrics["hunts"] == first["metrics"]["hunts"] + 2 and metrics["usd_total"] > first["metrics"]["usd_total"] + assert set(first["metrics"]) - {"tiebreak"} <= set(metrics) + + +def test_b_and_c_agreeing_flips_to_agreed_and_a_lone_a_is_rejected(tmp_path: Path) -> None: + root, a, b = _two_passes(tmp_path) + agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES) + c = _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"app.py::other"}) + report_metrics = _final(tmp_path, a, b, c)["metrics"] + report = json.loads((tmp_path / "final" / "agreed.json").read_text()) + assert [f["candidate"] for f in report["agreed"]] == ["app.py::other", "app.py::run"] + assert [f["candidate"] for f in report["disputed"]] == ["db.py::query"] + assert report_metrics["declared_sites_matched_agreed"] == 1 and report_metrics["tiebreak"]["flipped_to_agreed"] == 1 + + +def test_without_pass_c_the_a_b_rule_and_numbers_are_unchanged(tmp_path: Path) -> None: + _, a, b = _two_passes(tmp_path) + before = agree.run(tmp_path / "ab", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)["metrics"] + empty = _final(tmp_path, a, b, [])["metrics"] + assert before["tiebreak"] == {"asked": 0, "flipped_to_agreed": 0, "flipped_to_rejected": 0, "rule": agree.RULE_A_B} + assert empty["tiebreak"]["rule"] == agree.RULE_2_OF_3 and empty["tiebreak"]["asked"] == 0 + assert {k: v for k, v in empty.items() if k != "tiebreak"} == {k: v for k, v in before.items() if k != "tiebreak"} + + +def test_pass_c_alone_never_decides(tmp_path: Path) -> None: + root = _workspace(tmp_path) + a, b = tmp_path / "pass-a", tmp_path / "pass-b" + for out in (a, b): + _verify(root, out, HuntClient({"app.py::run"}), None, label=out.name[-1]) + c = tmp_path / "pass-c" + _verify(root, c, HuntClient({"app.py::other"}), None) # asked about everything, flags only what a and b rejected + metrics = _final(tmp_path, a, b, agree.load_pass(c))["metrics"] + report = json.loads((tmp_path / "final" / "agreed.json").read_text()) + assert [f["candidate"] for f in report["agreed"]] == ["app.py::run"] # 2 of 3 without c's vote + assert [f["candidate"] for f in report["disputed"]] == ["app.py::other"] # 1 of 3 + assert metrics["tiebreak"] == {"asked": 3, "flipped_to_agreed": 0, "flipped_to_rejected": 0, "rule": agree.RULE_2_OF_3} + + +def test_agree_main_reads_pass_c(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root, a, b = _two_passes(tmp_path) + agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES) + _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"db.py::query"}) + (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES)) + (tmp_path / "case.json").write_text(json.dumps(SITES)) + env = { + "OUSAST_OUTPUT_DIR": str(tmp_path / "final"), "OUSAST_INPUT_PASS_A": str(a / "units.jsonl"), + "OUSAST_INPUT_PASS_B": str(b / "units.jsonl"), "OUSAST_INPUT_PASS_C": str(tmp_path / "pass-c" / "units.jsonl"), + "OUSAST_INPUT_CANDIDATES": str(tmp_path / "candidates.json"), "OUSAST_INPUT_SITES": str(tmp_path / "case.json"), + } # fmt: skip + for key, value in env.items(): + monkeypatch.setenv(key, value) + assert agree.main() == 0 + metrics = json.loads((tmp_path / "final" / "summary.json").read_text())["metrics"] + assert metrics["agreed"] == 2 and metrics["tiebreak"]["flipped_to_agreed"] == 1