diff --git a/.kiro/specs/harnessx-removal/tasks.md b/.kiro/specs/harnessx-removal/tasks.md index 4a2837e..c4ed7a6 100644 --- a/.kiro/specs/harnessx-removal/tasks.md +++ b/.kiro/specs/harnessx-removal/tasks.md @@ -31,7 +31,7 @@ and passed its fixture test. Every task is a commit gated on the full suite's ow unchanged. - _Requirements: 6.2, 6.4_ -- [ ] 4. The loop as a plane Run +- [x] 4. The loop as a plane Run - Tasks `alerts`, `measure`, `propose`, `improve` (no Model, zero budget); `--loop` Run generation; fixed-pin Workspaces; the gate's verdict in `/loop-improve/gate.json`; adoption stays a maintainer commit. - Fixture tests with the fake ax; one live loop Run on the local ax recorded (no model spend beyond the verify diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index df40b79..cc4ed93 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,7 +1,7 @@ { - "modules": 135, + "modules": 137, "counts": { - "load_bearing": 126, + "load_bearing": 128, "standalone_capability": 9 }, "orphaned": [], @@ -27,7 +27,8 @@ "improve.evolve", "map_gate", "mcp", - "pairs" + "pairs", + "plane.tasks.loop" ] }, { @@ -232,6 +233,7 @@ "model.ladder", "model.report", "pairs", + "plane.tasks.alerts", "rank", "regress.candidate", "reports", @@ -258,7 +260,8 @@ "classification": "load_bearing", "reason": "entry point", "importers": [ - "cli" + "cli", + "plane.tasks.loop" ] }, { @@ -314,7 +317,8 @@ "importers": [ "cli", "improve.evolve", - "improve.memory" + "improve.memory", + "plane.tasks.loop" ] }, { @@ -322,7 +326,8 @@ "classification": "load_bearing", "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ - "improve" + "improve", + "plane.tasks.loop" ] }, { @@ -332,7 +337,8 @@ "importers": [ "improve", "improve.evolve", - "improve.memory" + "improve.memory", + "plane.tasks.loop" ] }, { @@ -341,7 +347,8 @@ "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", - "improve.evolve" + "improve.evolve", + "plane.tasks.loop" ] }, { @@ -351,7 +358,8 @@ "importers": [ "improve", "improve.evolve", - "improve.memory" + "improve.memory", + "plane.tasks.loop" ] }, { @@ -381,6 +389,7 @@ "map_gate", "model.scan", "pairs", + "plane.tasks.alerts", "preprocess", "push.runner", "stage_processors" @@ -666,7 +675,8 @@ "improve.evolve", "improve.memory", "model.candidates", - "pair_gate" + "pair_gate", + "plane.tasks.loop" ] }, { @@ -682,6 +692,7 @@ "plane.reconciler", "plane.router", "plane.runner", + "plane.tasks.loop", "plane.tasks.remember", "plane.tasks.verify", "plane.workspaces" @@ -740,6 +751,7 @@ "importers": [ "improve.memory", "plane.generate", + "plane.tasks.loop", "plane.tasks.remember" ] }, @@ -775,7 +787,9 @@ "importers": [ "cli", "plane.generate", + "plane.memory", "plane.tasks.agree", + "plane.tasks.alerts", "plane.tasks.remember" ] }, @@ -785,6 +799,20 @@ "reason": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match", "importers": [] }, + { + "module": "plane.tasks.alerts", + "classification": "load_bearing", + "reason": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free", + "importers": [] + }, + { + "module": "plane.tasks.loop", + "classification": "load_bearing", + "reason": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps", + "importers": [ + "plane.memory" + ] + }, { "module": "plane.tasks.remember", "classification": "load_bearing", @@ -799,6 +827,7 @@ "reason": "ai-service-plane Req 5.1/5.2: source-only functions and callers per file; runner integration pending", "importers": [ "plane.generate", + "plane.tasks.alerts", "plane.tasks.remember" ] }, @@ -830,6 +859,7 @@ "hunter_harness", "improve.evolve", "improve.validator", + "plane.tasks.loop", "scoring.project_score", "semantic.overlay", "stage_processors" @@ -865,6 +895,7 @@ "model.partitions", "model.scan", "pairs", + "plane.tasks.alerts", "plane.tasks.repo_facts", "push.runner", "rank", @@ -1045,6 +1076,7 @@ "improve.evolve", "map_gate", "pairs", + "plane.tasks.alerts", "stage_processors" ] }, @@ -1136,6 +1168,7 @@ "improve.memory", "improve.validator", "pairs", + "plane.tasks.loop", "stage_processors" ] }, diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/gate.json b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/gate.json new file mode 100644 index 0000000..266a1cf --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/gate.json @@ -0,0 +1,29 @@ +{ + "accepted": false, + "adoption": null, + "benchmark_edits": [], + "manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml", + "memory_edits": [], + "outcome": { + "accepted": false, + "degradations": [], + "edits": [], + "fp_after": 0.0, + "fp_before": 0.0, + "matched_after": 2, + "matched_before": 2, + "per_profile_after": {}, + "per_profile_before": {}, + "profile_regressions": [], + "profiles_under_minimum": [], + "reason": "no_proposals", + "recall_after": 1.0, + "recall_before": 1.0, + "round": 1, + "score_after": 73, + "score_before": 73 + }, + "proposals": 0, + "reason": "no_proposals", + "round": 1 +} diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/measure.json b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/measure.json new file mode 100644 index 0000000..ea77875 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/measure.json @@ -0,0 +1,140 @@ +{ + "inputs_bytes": { + "froxlor_admin_ip_second_order_sqli": 5943, + "lmdeploy_media_url_ssrf": 43730, + "wp_gopay_log_filter_sqli": 1947 + }, + "rows": { + "run": 87, + "snapshot": 119, + "union": 191 + }, + "run": "validation-46", + "run_metrics": { + "agreed": 4, + "alerts": { + "c-shell-exec": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "c-unsafe-memory-copy": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "python-shell-true": { + "fixed": 1, + "fixed_in_fix_range": 0, + "vulnerable": 1 + }, + "python-unsafe-deserialization": { + "fixed": 8, + "fixed_in_fix_range": 0, + "vulnerable": 8 + }, + "python-unsafe-eval": { + "fixed": 23, + "fixed_in_fix_range": 0, + "vulnerable": 23 + } + }, + "candidates": 5, + "cases": 3, + "declared_sites_agreed": 4, + "declared_sites_matched": 5, + "dispute_rate": { + "injection": { + "candidates": 4, + "disputed": 1, + "rate": 0.25 + }, + "untrusted_destination": { + "candidates": 1, + "disputed": 0, + "rate": 0.0 + } + }, + "disputed": 1, + "rejected": 0, + "tiebreaks": 3, + "usd": 0.130134, + "usd_per_candidate": 0.026027 + }, + "store_metrics": { + "agreed": 28, + "alerts": {}, + "candidates": 43, + "cases": 14, + "declared_sites_agreed": 16, + "declared_sites_matched": 19, + "dispute_rate": { + "injection": { + "candidates": 38, + "disputed": 5, + "rate": 0.1316 + }, + "untrusted_destination": { + "candidates": 5, + "disputed": 0, + "rate": 0.0 + } + }, + "disputed": 5, + "rejected": 10, + "tiebreaks": 10, + "usd": 0.856679, + "usd_per_candidate": 0.019923 + }, + "union_metrics": { + "agreed": 28, + "alerts": { + "c-shell-exec": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "c-unsafe-memory-copy": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "python-shell-true": { + "fixed": 1, + "fixed_in_fix_range": 0, + "vulnerable": 1 + }, + "python-unsafe-deserialization": { + "fixed": 8, + "fixed_in_fix_range": 0, + "vulnerable": 8 + }, + "python-unsafe-eval": { + "fixed": 23, + "fixed_in_fix_range": 0, + "vulnerable": 23 + } + }, + "candidates": 43, + "cases": 14, + "declared_sites_agreed": 16, + "declared_sites_matched": 19, + "dispute_rate": { + "injection": { + "candidates": 38, + "disputed": 5, + "rate": 0.1316 + }, + "untrusted_destination": { + "candidates": 5, + "disputed": 0, + "rate": 0.0 + } + }, + "disputed": 5, + "rejected": 10, + "tiebreaks": 10, + "usd": 0.856679, + "usd_per_candidate": 0.019923 + } +} diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/record.json b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/record.json new file mode 100644 index 0000000..077f4ea --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/record.json @@ -0,0 +1,335 @@ +{ + "attribution_loop_tasks": [ + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "memory-snapshot", + "usd": null + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "froxlor-admin-ip-second-order-sqli-alerts", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "froxlor-admin-ip-second-order-sqli-remember", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "lmdeploy-media-url-ssrf-alerts", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "lmdeploy-media-url-ssrf-remember", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "wp-gopay-log-filter-sqli-alerts", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "wp-gopay-log-filter-sqli-remember", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "loop-measure", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "loop-propose", + "usd": 0 + }, + { + "cache_hit": null, + "calls": 0, + "model": null, + "output": null, + "prompt": null, + "status": "done", + "task": "loop-improve", + "usd": 0 + } + ], + "cases": [ + "wp-gopay-log-filter-sqli", + "lmdeploy-media-url-ssrf", + "froxlor-admin-ip-second-order-sqli" + ], + "new_spend_usd": 0.0, + "note": "harnessx-removal task 4: one live loop Run on the local ax (kind), 2026-09-30. The Run is validation-46 restricted to three cases (set.json) plus the loop; the cases' chain tasks (facts, va, vb, agree, vc, final) were already done in ~/ousast-results/plane/validation-46 and were not rerun. Runner image localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9; loop-propose/loop-improve bound github.com/norandom/OpenUltraSAST at 1ad472c (origin/main, whose benchmarks/ manifests, fixtures and pairs equal the commit under test); the code under test is the image's package.", + "tasks": { + "froxlor-admin-ip-second-order-sqli-alerts": { + "finished": "2026-09-30T09:51:22+00:00", + "started": "2026-09-30T09:50:49+00:00", + "status": "done", + "summary": { + "alerts": { + "fixed": 0, + "vulnerable": 0 + }, + "calls": 0, + "model": null, + "read": { + "fixed": { + "bytes": 4256572, + "files": 366 + }, + "vulnerable": { + "bytes": 4256094, + "files": 366 + } + }, + "status": "done", + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 + } + }, + "froxlor-admin-ip-second-order-sqli-remember": { + "finished": "2026-09-30T09:51:51+00:00", + "started": "2026-09-30T09:51:22+00:00", + "status": "done", + "summary": { + "calls": 0, + "facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it", + "model": null, + "rows": 9, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + } + }, + "lmdeploy-media-url-ssrf-alerts": { + "finished": "2026-09-30T09:52:27+00:00", + "started": "2026-09-30T09:51:22+00:00", + "status": "done", + "summary": { + "alerts": { + "fixed": 36, + "vulnerable": 36 + }, + "calls": 0, + "model": null, + "read": { + "fixed": { + "bytes": 8353172, + "files": 1084 + }, + "vulnerable": { + "bytes": 8340364, + "files": 1084 + } + }, + "status": "done", + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 + } + }, + "lmdeploy-media-url-ssrf-remember": { + "finished": "2026-09-30T09:52:56+00:00", + "started": "2026-09-30T09:52:27+00:00", + "status": "done", + "summary": { + "calls": 0, + "facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it", + "model": null, + "rows": 75, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + } + }, + "loop-improve": { + "finished": "2026-09-30T09:54:44+00:00", + "started": "2026-09-30T09:54:17+00:00", + "status": "done", + "summary": { + "accepted": false, + "calls": 0, + "edits": 0, + "memory_edits": 0, + "model": null, + "reason": "no_proposals", + "status": "done", + "step": "improve", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + } + }, + "loop-measure": { + "finished": "2026-09-30T09:53:50+00:00", + "started": "2026-09-30T09:53:27+00:00", + "status": "done", + "summary": { + "calls": 0, + "model": null, + "rows": { + "run": 87, + "snapshot": 119, + "union": 191 + }, + "status": "done", + "step": "measure", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + } + }, + "loop-propose": { + "finished": "2026-09-30T09:54:17+00:00", + "started": "2026-09-30T09:53:50+00:00", + "status": "done", + "summary": { + "calls": 0, + "excluded": 0, + "kept": 191, + "model": null, + "proposals": 0, + "rows": 191, + "status": "done", + "step": "propose", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + } + }, + "memory-snapshot": { + "finished": "2026-09-30T09:50:48+00:00", + "started": null, + "status": "done", + "summary": { + "calls": 0, + "model": null, + "reused": { + "excluded": { + "gated_case": {}, + "holdout_pair": {}, + "population": {}, + "rows": 0 + }, + "guard": { + "catalog": "benchmarks/pairs/catalog.toml", + "manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml", + "populations": [] + }, + "index_digest": "6fa3c76576e6706a8b5b439611c15f1337754a2243ab95d6d9b45899b06c5215", + "index_entries": 15, + "kept": 119, + "rows": 119, + "store": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory" + }, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": null + } + }, + "wp-gopay-log-filter-sqli-alerts": { + "finished": "2026-09-30T09:52:59+00:00", + "started": "2026-09-30T09:52:27+00:00", + "status": "done", + "summary": { + "alerts": { + "fixed": 0, + "vulnerable": 0 + }, + "calls": 0, + "model": null, + "read": { + "fixed": { + "bytes": 150539, + "files": 19 + }, + "vulnerable": { + "bytes": 150528, + "files": 19 + } + }, + "status": "done", + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 + } + }, + "wp-gopay-log-filter-sqli-remember": { + "finished": "2026-09-30T09:53:27+00:00", + "started": "2026-09-30T09:52:59+00:00", + "status": "done", + "summary": { + "calls": 0, + "facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it", + "model": null, + "rows": 3, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + } + } + } +} diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/run.yaml b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/run.yaml new file mode 100644 index 0000000..367d49a --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/run.yaml @@ -0,0 +1,320 @@ +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set /tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr4/set/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --loop --loop-commit 1ad472c68d759e4ef010bd7a2e23532b1de538a9 --loop-repo https://github.com/norandom/OpenUltraSAST.git --loop-manifest benchmarks/manifests/java-spring-boot-vulnerable.toml`; do not edit, regenerate. +# 3 cases, 5 candidates (5 kept by the recorded triage); usd ceiling 0.33. +apiVersion: openultrasast.io/v1alpha1 +kind: Run +metadata: + name: validation-46 + annotations: + openultrasast.io/population: population-v2 + openultrasast.io/split: validation +spec: + tasks: + - name: froxlor-admin-ip-second-order-sqli-facts + task: repo-facts-froxlor-admin-ip-second-order-sqli + outputs: + - facts.json + budget: &id001 + usd: 0 + calls: 0 + serialize: repo-facts + - name: froxlor-admin-ip-second-order-sqli-va + task: verify-a-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-vb + task: verify-b-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-agree + task: agree-froxlor-admin-ip-second-order-sqli + inputs: + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: *id001 + - name: lmdeploy-media-url-ssrf-facts + task: repo-facts-lmdeploy-media-url-ssrf + outputs: + - facts.json + budget: &id002 + usd: 0 + calls: 0 + serialize: repo-facts + - name: lmdeploy-media-url-ssrf-va + task: verify-a-lmdeploy-media-url-ssrf + inputs: + facts: lmdeploy-media-url-ssrf-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: lmdeploy-media-url-ssrf-vb + task: verify-b-lmdeploy-media-url-ssrf + inputs: + facts: lmdeploy-media-url-ssrf-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: lmdeploy-media-url-ssrf-agree + task: agree-lmdeploy-media-url-ssrf + inputs: + pass_a: lmdeploy-media-url-ssrf-va/units.jsonl + pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: *id002 + - name: wp-gopay-log-filter-sqli-facts + task: repo-facts-wp-gopay-log-filter-sqli + outputs: + - facts.json + budget: &id003 + usd: 0 + calls: 0 + serialize: repo-facts + - name: wp-gopay-log-filter-sqli-va + task: verify-a-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-vb + task: verify-b-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-agree + task: agree-wp-gopay-log-filter-sqli + inputs: + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: *id003 + - name: froxlor-admin-ip-second-order-sqli-vc + task: verify-c-froxlor-admin-ip-second-order-sqli + dependsOn: + - froxlor-admin-ip-second-order-sqli-agree + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + only: froxlor-admin-ip-second-order-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-final + task: agree-froxlor-admin-ip-second-order-sqli + inputs: + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: lmdeploy-media-url-ssrf-vc + task: verify-c-lmdeploy-media-url-ssrf + dependsOn: + - lmdeploy-media-url-ssrf-agree + inputs: + facts: lmdeploy-media-url-ssrf-facts/facts.json + only: lmdeploy-media-url-ssrf-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: lmdeploy-media-url-ssrf-final + task: agree-lmdeploy-media-url-ssrf + inputs: + pass_a: lmdeploy-media-url-ssrf-va/units.jsonl + pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl + pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: wp-gopay-log-filter-sqli-vc + task: verify-c-wp-gopay-log-filter-sqli + dependsOn: + - wp-gopay-log-filter-sqli-agree + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + only: wp-gopay-log-filter-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-final + task: agree-wp-gopay-log-filter-sqli + inputs: + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: froxlor-admin-ip-second-order-sqli-alerts + task: alerts-froxlor-admin-ip-second-order-sqli + outputs: + - alerts.jsonl + budget: + usd: 0 + calls: 0 + serialize: engine + - name: froxlor-admin-ip-second-order-sqli-remember + task: remember-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + facts_summary: froxlor-admin-ip-second-order-sqli-facts/summary.json + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + summary_a: froxlor-admin-ip-second-order-sqli-va/summary.json + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + summary_b: froxlor-admin-ip-second-order-sqli-vb/summary.json + pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl + summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json + agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json + alerts: froxlor-admin-ip-second-order-sqli-alerts/alerts.jsonl + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: lmdeploy-media-url-ssrf-alerts + task: alerts-lmdeploy-media-url-ssrf + outputs: + - alerts.jsonl + budget: + usd: 0 + calls: 0 + serialize: engine + - name: lmdeploy-media-url-ssrf-remember + task: remember-lmdeploy-media-url-ssrf + inputs: + facts: lmdeploy-media-url-ssrf-facts/facts.json + facts_summary: lmdeploy-media-url-ssrf-facts/summary.json + pass_a: lmdeploy-media-url-ssrf-va/units.jsonl + summary_a: lmdeploy-media-url-ssrf-va/summary.json + pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl + summary_b: lmdeploy-media-url-ssrf-vb/summary.json + pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl + summary_c: lmdeploy-media-url-ssrf-vc/summary.json + agreed: lmdeploy-media-url-ssrf-final/agreed.json + alerts: lmdeploy-media-url-ssrf-alerts/alerts.jsonl + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: wp-gopay-log-filter-sqli-alerts + task: alerts-wp-gopay-log-filter-sqli + outputs: + - alerts.jsonl + budget: + usd: 0 + calls: 0 + serialize: engine + - name: wp-gopay-log-filter-sqli-remember + task: remember-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + facts_summary: wp-gopay-log-filter-sqli-facts/summary.json + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + summary_a: wp-gopay-log-filter-sqli-va/summary.json + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + summary_b: wp-gopay-log-filter-sqli-vb/summary.json + pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl + summary_c: wp-gopay-log-filter-sqli-vc/summary.json + agreed: wp-gopay-log-filter-sqli-final/agreed.json + alerts: wp-gopay-log-filter-sqli-alerts/alerts.jsonl + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: memory-snapshot + task: memory-snapshot + outputs: + - rows.jsonl + - index.json + budget: + usd: 0 + calls: 0 + - name: loop-measure + task: loop-measure + inputs: + remember_froxlor-admin-ip-second-order-sqli: froxlor-admin-ip-second-order-sqli-remember/memory.jsonl + remember_lmdeploy-media-url-ssrf: lmdeploy-media-url-ssrf-remember/memory.jsonl + remember_wp-gopay-log-filter-sqli: wp-gopay-log-filter-sqli-remember/memory.jsonl + snapshot: memory-snapshot/rows.jsonl + outputs: + - measure.json + - rows.jsonl + budget: + usd: 0 + calls: 0 + - name: loop-propose + task: loop-propose + inputs: + measure: loop-measure/measure.json + rows: loop-measure/rows.jsonl + index: memory-snapshot/index.json + outputs: + - proposals.jsonl + - memory_proposals.jsonl + - signals.json + budget: + usd: 0 + calls: 0 + - name: loop-improve + task: loop-improve + inputs: + proposals: loop-propose/proposals.jsonl + outputs: + - gate.json + - journal.json + budget: + usd: 0 + calls: 0 + serialize: engine diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/set.json b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/set.json new file mode 100644 index 0000000..689b854 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/set.json @@ -0,0 +1,28 @@ +{ + "wp-gopay-log-filter-sqli": [ + [ + "admin/views/log.php", + "" + ] + ], + "lmdeploy-media-url-ssrf": [ + [ + "lmdeploy/vl/media/connection.py", + "_load_http_url" + ] + ], + "froxlor-admin-ip-second-order-sqli": [ + [ + "lib/Froxlor/Api/Commands/Domains.php", + "validateIpAddresses" + ], + [ + "lib/Froxlor/Api/Commands/IpsAndPorts.php", + "listing" + ], + [ + "lib/Froxlor/Api/Commands/IpsAndPorts.php", + "listingCount" + ] + ] +} \ No newline at end of file diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/signals.json b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/signals.json new file mode 100644 index 0000000..90ee97f --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/signals.json @@ -0,0 +1,8 @@ +{ + "advisory": true, + "disputed_families": [], + "thresholds": { + "min_candidates": 5, + "min_dispute_rate": 0.3 + } +} diff --git a/benchmarks/measurements/2026-09-30-harnessx-removal-loop/snapshot-index.json b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/snapshot-index.json new file mode 100644 index 0000000..80b33ea --- /dev/null +++ b/benchmarks/measurements/2026-09-30-harnessx-removal-loop/snapshot-index.json @@ -0,0 +1,18 @@ +{ + "excluded": { + "gated_case": {}, + "holdout_pair": {}, + "population": {}, + "rows": 0 + }, + "guard": { + "catalog": "benchmarks/pairs/catalog.toml", + "manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml", + "populations": [] + }, + "index_digest": "6fa3c76576e6706a8b5b439611c15f1337754a2243ab95d6d9b45899b06c5215", + "index_entries": 15, + "kept": 119, + "rows": 119, + "store": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory" +} diff --git a/plane/runs/validation-46.yaml b/plane/runs/validation-46.yaml index 20f3977..a6ea9a2 100644 --- a/plane/runs/validation-46.yaml +++ b/plane/runs/validation-46.yaml @@ -939,3 +939,258 @@ spec: budget: usd: 0 calls: 0 + - name: budibase-mongo-template-nosqli-remember + task: remember-budibase-mongo-template-nosqli + inputs: + facts: budibase-mongo-template-nosqli-facts/facts.json + facts_summary: budibase-mongo-template-nosqli-facts/summary.json + pass_a: budibase-mongo-template-nosqli-va/units.jsonl + summary_a: budibase-mongo-template-nosqli-va/summary.json + pass_b: budibase-mongo-template-nosqli-vb/units.jsonl + summary_b: budibase-mongo-template-nosqli-vb/summary.json + pass_c: budibase-mongo-template-nosqli-vc/units.jsonl + summary_c: budibase-mongo-template-nosqli-vc/summary.json + agreed: budibase-mongo-template-nosqli-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: cct-studio-server-cmdi-remember + task: remember-cct-studio-server-cmdi + inputs: + facts: cct-studio-server-cmdi-facts/facts.json + facts_summary: cct-studio-server-cmdi-facts/summary.json + pass_a: cct-studio-server-cmdi-va/units.jsonl + summary_a: cct-studio-server-cmdi-va/summary.json + pass_b: cct-studio-server-cmdi-vb/units.jsonl + summary_b: cct-studio-server-cmdi-vb/summary.json + pass_c: cct-studio-server-cmdi-vc/units.jsonl + summary_c: cct-studio-server-cmdi-vc/summary.json + agreed: cct-studio-server-cmdi-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: contextforge-prompt-ssti-remember + task: remember-contextforge-prompt-ssti + inputs: + facts: contextforge-prompt-ssti-facts/facts.json + facts_summary: contextforge-prompt-ssti-facts/summary.json + pass_a: contextforge-prompt-ssti-va/units.jsonl + summary_a: contextforge-prompt-ssti-va/summary.json + pass_b: contextforge-prompt-ssti-vb/units.jsonl + summary_b: contextforge-prompt-ssti-vb/summary.json + pass_c: contextforge-prompt-ssti-vc/units.jsonl + summary_c: contextforge-prompt-ssti-vc/summary.json + agreed: contextforge-prompt-ssti-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: froxlor-admin-ip-second-order-sqli-remember + task: remember-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + facts_summary: froxlor-admin-ip-second-order-sqli-facts/summary.json + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + summary_a: froxlor-admin-ip-second-order-sqli-va/summary.json + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + summary_b: froxlor-admin-ip-second-order-sqli-vb/summary.json + pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl + summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json + agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: fuxa-tdengine-escape-sqli-remember + task: remember-fuxa-tdengine-escape-sqli + inputs: + facts: fuxa-tdengine-escape-sqli-facts/facts.json + facts_summary: fuxa-tdengine-escape-sqli-facts/summary.json + pass_a: fuxa-tdengine-escape-sqli-va/units.jsonl + summary_a: fuxa-tdengine-escape-sqli-va/summary.json + pass_b: fuxa-tdengine-escape-sqli-vb/units.jsonl + summary_b: fuxa-tdengine-escape-sqli-vb/summary.json + pass_c: fuxa-tdengine-escape-sqli-vc/units.jsonl + summary_c: fuxa-tdengine-escape-sqli-vc/summary.json + agreed: fuxa-tdengine-escape-sqli-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: lmdeploy-media-url-ssrf-remember + task: remember-lmdeploy-media-url-ssrf + inputs: + facts: lmdeploy-media-url-ssrf-facts/facts.json + facts_summary: lmdeploy-media-url-ssrf-facts/summary.json + pass_a: lmdeploy-media-url-ssrf-va/units.jsonl + summary_a: lmdeploy-media-url-ssrf-va/summary.json + pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl + summary_b: lmdeploy-media-url-ssrf-vb/summary.json + pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl + summary_c: lmdeploy-media-url-ssrf-vc/summary.json + agreed: lmdeploy-media-url-ssrf-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: medplum-external-auth-open-redirect-remember + task: remember-medplum-external-auth-open-redirect + inputs: + facts: medplum-external-auth-open-redirect-facts/facts.json + facts_summary: medplum-external-auth-open-redirect-facts/summary.json + pass_a: medplum-external-auth-open-redirect-va/units.jsonl + summary_a: medplum-external-auth-open-redirect-va/summary.json + pass_b: medplum-external-auth-open-redirect-vb/units.jsonl + summary_b: medplum-external-auth-open-redirect-vb/summary.json + pass_c: medplum-external-auth-open-redirect-vc/units.jsonl + summary_c: medplum-external-auth-open-redirect-vc/summary.json + agreed: medplum-external-auth-open-redirect-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: pgadmin-maintenance-sqli-remember + task: remember-pgadmin-maintenance-sqli + inputs: + facts: pgadmin-maintenance-sqli-facts/facts.json + facts_summary: pgadmin-maintenance-sqli-facts/summary.json + pass_a: pgadmin-maintenance-sqli-va/units.jsonl + summary_a: pgadmin-maintenance-sqli-va/summary.json + pass_b: pgadmin-maintenance-sqli-vb/units.jsonl + summary_b: pgadmin-maintenance-sqli-vb/summary.json + pass_c: pgadmin-maintenance-sqli-vc/units.jsonl + summary_c: pgadmin-maintenance-sqli-vc/summary.json + agreed: pgadmin-maintenance-sqli-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: qwed-sympy-eval-remember + task: remember-qwed-sympy-eval + inputs: + facts: qwed-sympy-eval-facts/facts.json + facts_summary: qwed-sympy-eval-facts/summary.json + pass_a: qwed-sympy-eval-va/units.jsonl + summary_a: qwed-sympy-eval-va/summary.json + pass_b: qwed-sympy-eval-vb/units.jsonl + summary_b: qwed-sympy-eval-vb/summary.json + pass_c: qwed-sympy-eval-vc/units.jsonl + summary_c: qwed-sympy-eval-vc/summary.json + agreed: qwed-sympy-eval-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: reactpress-findall-column-sqli-remember + task: remember-reactpress-findall-column-sqli + inputs: + facts: reactpress-findall-column-sqli-facts/facts.json + facts_summary: reactpress-findall-column-sqli-facts/summary.json + pass_a: reactpress-findall-column-sqli-va/units.jsonl + summary_a: reactpress-findall-column-sqli-va/summary.json + pass_b: reactpress-findall-column-sqli-vb/units.jsonl + summary_b: reactpress-findall-column-sqli-vb/summary.json + pass_c: reactpress-findall-column-sqli-vc/units.jsonl + summary_c: reactpress-findall-column-sqli-vc/summary.json + agreed: reactpress-findall-column-sqli-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: unleash-addon-webhook-ssrf-remember + task: remember-unleash-addon-webhook-ssrf + inputs: + facts: unleash-addon-webhook-ssrf-facts/facts.json + facts_summary: unleash-addon-webhook-ssrf-facts/summary.json + pass_a: unleash-addon-webhook-ssrf-va/units.jsonl + summary_a: unleash-addon-webhook-ssrf-va/summary.json + pass_b: unleash-addon-webhook-ssrf-vb/units.jsonl + summary_b: unleash-addon-webhook-ssrf-vb/summary.json + pass_c: unleash-addon-webhook-ssrf-vc/units.jsonl + summary_c: unleash-addon-webhook-ssrf-vc/summary.json + agreed: unleash-addon-webhook-ssrf-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: winml-cli-serve-cors-remember + task: remember-winml-cli-serve-cors + inputs: + facts: winml-cli-serve-cors-facts/facts.json + facts_summary: winml-cli-serve-cors-facts/summary.json + pass_a: winml-cli-serve-cors-va/units.jsonl + summary_a: winml-cli-serve-cors-va/summary.json + pass_b: winml-cli-serve-cors-vb/units.jsonl + summary_b: winml-cli-serve-cors-vb/summary.json + pass_c: winml-cli-serve-cors-vc/units.jsonl + summary_c: winml-cli-serve-cors-vc/summary.json + agreed: winml-cli-serve-cors-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: wp-directorist-avatar-ssrf-remember + task: remember-wp-directorist-avatar-ssrf + inputs: + facts: wp-directorist-avatar-ssrf-facts/facts.json + facts_summary: wp-directorist-avatar-ssrf-facts/summary.json + pass_a: wp-directorist-avatar-ssrf-va/units.jsonl + summary_a: wp-directorist-avatar-ssrf-va/summary.json + pass_b: wp-directorist-avatar-ssrf-vb/units.jsonl + summary_b: wp-directorist-avatar-ssrf-vb/summary.json + pass_c: wp-directorist-avatar-ssrf-vc/units.jsonl + summary_c: wp-directorist-avatar-ssrf-vc/summary.json + agreed: wp-directorist-avatar-ssrf-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: wp-gopay-log-filter-sqli-remember + task: remember-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + facts_summary: wp-gopay-log-filter-sqli-facts/summary.json + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + summary_a: wp-gopay-log-filter-sqli-va/summary.json + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + summary_b: wp-gopay-log-filter-sqli-vb/summary.json + pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl + summary_c: wp-gopay-log-filter-sqli-vc/summary.json + agreed: wp-gopay-log-filter-sqli-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: wp-groundhogg-confirm-redirect-remember + task: remember-wp-groundhogg-confirm-redirect + inputs: + facts: wp-groundhogg-confirm-redirect-facts/facts.json + facts_summary: wp-groundhogg-confirm-redirect-facts/summary.json + pass_a: wp-groundhogg-confirm-redirect-va/units.jsonl + summary_a: wp-groundhogg-confirm-redirect-va/summary.json + pass_b: wp-groundhogg-confirm-redirect-vb/units.jsonl + summary_b: wp-groundhogg-confirm-redirect-vb/summary.json + pass_c: wp-groundhogg-confirm-redirect-vc/units.jsonl + summary_c: wp-groundhogg-confirm-redirect-vc/summary.json + agreed: wp-groundhogg-confirm-redirect-final/agreed.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 diff --git a/plane/tasks/agree.yaml b/plane/tasks/agree.yaml index 337a40f..f3920bf 100644 --- a/plane/tasks/agree.yaml +++ b/plane/tasks/agree.yaml @@ -7,7 +7,7 @@ metadata: name: agree atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617" + image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" command: ["agree"] resources: requests: diff --git a/plane/tasks/alerts.yaml b/plane/tasks/alerts.yaml new file mode 100644 index 0000000..d17bd39 --- /dev/null +++ b/plane/tasks/alerts.yaml @@ -0,0 +1,18 @@ +# alerts (harnessx-removal Req 6.3, design section 6): quick-mode rules (enabled and shadow) on a case's vulnerable +# and fixed pins, each alert attributed to its enclosing function and marked inside or outside the fix. No model. +# Heavy like repo-facts (a full-tree scan per pin): the Run serializes it. The image is digest-pinned. +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: alerts + atespace: default +spec: + image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + command: ["alerts"] + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1280Mi" diff --git a/plane/tasks/loop.yaml b/plane/tasks/loop.yaml new file mode 100644 index 0000000..d936d9e --- /dev/null +++ b/plane/tasks/loop.yaml @@ -0,0 +1,18 @@ +# loop (harnessx-removal Req 6.3, design section 6): the improvement loop's singletons -- the generator appends the +# step (snapshot, measure, propose, improve) to the command. No model, budget {usd: 0, calls: 0}; improve replays the +# benchmark gate unchanged and delivers its verdict as gate.json. The image is digest-pinned. +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: loop + atespace: default +spec: + image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + command: ["loop"] + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1280Mi" diff --git a/plane/tasks/remember.yaml b/plane/tasks/remember.yaml new file mode 100644 index 0000000..d11e9cb --- /dev/null +++ b/plane/tasks/remember.yaml @@ -0,0 +1,18 @@ +# remember (harnessx-removal Req 6.1, design section 4): a case's delivered artifacts as memory rows. No model. +# The template of the per-case Tasks (bound to the case's vulnerable Workspace for the repository and pin the rows +# carry); the host ingests the delivered memory.jsonl after the Run. The image is digest-pinned. +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember + atespace: default +spec: + image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + command: ["remember"] + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "512Mi" diff --git a/plane/tasks/repo-facts.yaml b/plane/tasks/repo-facts.yaml index 0b386d7..3d642f4 100644 --- a/plane/tasks/repo-facts.yaml +++ b/plane/tasks/repo-facts.yaml @@ -8,7 +8,7 @@ metadata: name: repo-facts atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617" + image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" command: ["repo-facts"] resources: requests: diff --git a/plane/tasks/validation-46.yaml b/plane/tasks/validation-46.yaml index 7fc8cee..48a0231 100644 --- a/plane/tasks/validation-46.yaml +++ b/plane/tasks/validation-46.yaml @@ -5,9 +5,9 @@ metadata: name: repo-facts-budibase-mongo-template-nosqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}' + openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -36,7 +36,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -67,7 +67,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -96,7 +96,7 @@ metadata: name: agree-budibase-mongo-template-nosqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -123,9 +123,9 @@ metadata: name: repo-facts-cct-studio-server-cmdi atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}' + openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -154,7 +154,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -185,7 +185,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -214,7 +214,7 @@ metadata: name: agree-cct-studio-server-cmdi atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -241,9 +241,9 @@ metadata: name: repo-facts-contextforge-prompt-ssti atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}' + openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -272,7 +272,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -303,7 +303,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -332,7 +332,7 @@ metadata: name: agree-contextforge-prompt-ssti atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -359,9 +359,9 @@ metadata: name: repo-facts-froxlor-admin-ip-second-order-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}' + openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -390,7 +390,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -421,7 +421,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -450,7 +450,7 @@ metadata: name: agree-froxlor-admin-ip-second-order-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -477,9 +477,9 @@ metadata: name: repo-facts-fuxa-tdengine-escape-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}' + openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -508,7 +508,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -539,7 +539,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -568,7 +568,7 @@ metadata: name: agree-fuxa-tdengine-escape-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -595,9 +595,9 @@ metadata: name: repo-facts-lmdeploy-media-url-ssrf atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}' + openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -626,7 +626,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -657,7 +657,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -686,7 +686,7 @@ metadata: name: agree-lmdeploy-media-url-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -713,9 +713,9 @@ metadata: name: repo-facts-medplum-external-auth-open-redirect atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}' + openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -744,7 +744,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -775,7 +775,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -804,7 +804,7 @@ metadata: name: agree-medplum-external-auth-open-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -831,9 +831,9 @@ metadata: name: repo-facts-pgadmin-maintenance-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}' + openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -862,7 +862,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -893,7 +893,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -922,7 +922,7 @@ metadata: name: agree-pgadmin-maintenance-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -949,9 +949,9 @@ metadata: name: repo-facts-qwed-sympy-eval atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}' + openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -980,7 +980,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1011,7 +1011,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1040,7 +1040,7 @@ metadata: name: agree-qwed-sympy-eval atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1067,9 +1067,9 @@ metadata: name: repo-facts-reactpress-findall-column-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}' + openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -1098,7 +1098,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1129,7 +1129,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1158,7 +1158,7 @@ metadata: name: agree-reactpress-findall-column-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1185,9 +1185,9 @@ metadata: name: repo-facts-unleash-addon-webhook-ssrf atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}' + openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -1216,7 +1216,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1247,7 +1247,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1276,7 +1276,7 @@ metadata: name: agree-unleash-addon-webhook-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1303,9 +1303,9 @@ metadata: name: repo-facts-winml-cli-serve-cors atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}' + openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -1334,7 +1334,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1365,7 +1365,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1394,7 +1394,7 @@ metadata: name: agree-winml-cli-serve-cors atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1421,9 +1421,9 @@ metadata: name: repo-facts-wp-directorist-avatar-ssrf atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}' + openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -1452,7 +1452,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1483,7 +1483,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1512,7 +1512,7 @@ metadata: name: agree-wp-directorist-avatar-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1539,9 +1539,9 @@ metadata: name: repo-facts-wp-gopay-log-filter-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}' + openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -1570,7 +1570,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1601,7 +1601,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1630,7 +1630,7 @@ metadata: name: agree-wp-gopay-log-filter-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1657,9 +1657,9 @@ metadata: name: repo-facts-wp-groundhogg-confirm-redirect atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}' + openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}' spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - repo-facts env: @@ -1688,7 +1688,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1719,7 +1719,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1748,7 +1748,7 @@ metadata: name: agree-wp-groundhogg-confirm-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - agree env: @@ -1777,7 +1777,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1808,7 +1808,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1839,7 +1839,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1870,7 +1870,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1901,7 +1901,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1932,7 +1932,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1963,7 +1963,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -1994,7 +1994,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2025,7 +2025,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2056,7 +2056,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2087,7 +2087,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2118,7 +2118,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2149,7 +2149,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2180,7 +2180,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2211,7 +2211,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 command: - verify env: @@ -2233,3 +2233,378 @@ spec: path: /workspace/case - name: wp-groundhogg-confirm-redirect-inputs path: /workspace/inputs +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-budibase-mongo-template-nosqli + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: budibase-mongo-template-nosqli-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-cct-studio-server-cmdi + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: cct-studio-server-cmdi-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-contextforge-prompt-ssti + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: contextforge-prompt-ssti-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-froxlor-admin-ip-second-order-sqli + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: froxlor-admin-ip-second-order-sqli-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-fuxa-tdengine-escape-sqli + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: fuxa-tdengine-escape-sqli-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-lmdeploy-media-url-ssrf + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: lmdeploy-media-url-ssrf-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-medplum-external-auth-open-redirect + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: medplum-external-auth-open-redirect-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-pgadmin-maintenance-sqli + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: pgadmin-maintenance-sqli-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-qwed-sympy-eval + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: qwed-sympy-eval-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-reactpress-findall-column-sqli + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: reactpress-findall-column-sqli-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-unleash-addon-webhook-ssrf + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: unleash-addon-webhook-ssrf-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-winml-cli-serve-cors + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: winml-cli-serve-cors-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-wp-directorist-avatar-ssrf + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: wp-directorist-avatar-ssrf-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-wp-gopay-log-filter-sqli + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: wp-gopay-log-filter-sqli-vulnerable + path: /workspace/case +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: remember-wp-groundhogg-confirm-redirect + atespace: default +spec: + image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + command: + - remember + env: + - name: OUSAST_POPULATION + value: population-v2 + - name: OUSAST_SPLIT + value: validation + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 512Mi + workspaces: + - name: wp-groundhogg-confirm-redirect-vulnerable + path: /workspace/case diff --git a/plane/tasks/verify.yaml b/plane/tasks/verify.yaml index e94cf02..00721a7 100644 --- a/plane/tasks/verify.yaml +++ b/plane/tasks/verify.yaml @@ -10,7 +10,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: "localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617" + image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" command: ["verify"] resources: requests: diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 6669d74..3c7e21a 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -284,6 +284,16 @@ def main(argv: list[str] | None = None) -> int: plane_ws.add_argument( "--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)" ) + plane_ws.add_argument( + "--loop", action="store_true", + help="with --validation-set: append the improvement loop (alerts per case, then snapshot, measure, propose, improve)", + ) # fmt: skip + plane_ws.add_argument("--loop-commit", help="with --loop: the commit under test the gate runs at (default: git rev-parse HEAD)") + plane_ws.add_argument("--loop-repo", help="with --loop: this repository's URL (default: git remote get-url origin)") + plane_ws.add_argument("--loop-manifest", help="with --loop: the benchmark manifest the proposals are gated on") + plane_ws.add_argument( + "--qualify-population", action="append", default=[], metavar="NAME", help="with --loop: a population whose rows never propose" + ) args = parser.parse_args(argv) # Diagnosis needs the stage costs, and nothing configures logging, so the default root level of @@ -1382,7 +1392,7 @@ def _plane_memory( assert run_manifest is not None seeded = memory.seed(run_manifest, store) if seeded: - print(f"memory {store.describe()}: facts reused for {len(seeded)} tasks: {', '.join(seeded)}") + print(f"memory {store.describe()}: seeded {len(seeded)} tasks (reused facts, loop snapshot): {', '.join(seeded)}") return 0 annotations: dict[str, str] = {} if run_manifest is not None: @@ -1403,7 +1413,7 @@ def _plane_memory( def _plane_increment(args: argparse.Namespace) -> int: - from .plane.generate import increment + from .plane.generate import LOOP_MANIFEST, Loop, increment if args.candidates is None or args.triage is None: print("--validation-set needs --candidates and --triage", file=sys.stderr) @@ -1417,9 +1427,18 @@ def _plane_increment(args: argparse.Namespace) -> int: ) if args.runner_image is not None: command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}" + loop = None + if args.loop: + git = lambda *a: subprocess.run(["git", *a], capture_output=True, text=True, check=True).stdout.strip() # noqa: E731 + loop = Loop( + repo=args.loop_repo or git("remote", "get-url", "origin"), commit=args.loop_commit or git("rev-parse", "HEAD"), + manifest=args.loop_manifest or LOOP_MANIFEST, populations=tuple(args.qualify_population), + ) # fmt: skip + command += f" --loop --loop-commit {loop.commit} --loop-repo {loop.repo} --loop-manifest {loop.manifest}" + command += "".join(f" --qualify-population {p}" for p in loop.populations) written = increment( args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command, - repos=args.repos, runner_image=args.runner_image, + repos=args.repos, runner_image=args.runner_image, loop=loop, ) # fmt: skip print(f"{len(written)} manifests written under {args.plane} (Run {args.run})") return 0 diff --git a/src/openultrasast/improve/memory.py b/src/openultrasast/improve/memory.py index aea27f5..d5c0987 100644 --- a/src/openultrasast/improve/memory.py +++ b/src/openultrasast/improve/memory.py @@ -145,7 +145,10 @@ class _View: @classmethod def of(cls, records: Sequence[Record]) -> _View: + """One alert per (repo, pin, rule, path, line, function, pin role): every loop run re-alerts an unchanged pin under a + new run name, and a repeat is the same evidence, not more of it.""" view = cls() + seen: set[tuple[object, ...]] = set() for rec in records: row = rec.row if row["kind"] == "verdict": @@ -153,7 +156,10 @@ class _View: if row.get("site_match") is True: view.sites.add((row["repo"], str(row.get("candidate")))) elif row["kind"] == "alert" and row.get("rule_id"): - view.alerts.setdefault(str(row["rule_id"]), []).append(rec) + place = (row["repo"], row["pin"], row["rule_id"], *(row.get(k) for k in ("path", "line", "function", "pin_role"))) + if place not in seen: + seen.add(place) + view.alerts.setdefault(str(row["rule_id"]), []).append(rec) return view def verdict(self, alert: Mapping[str, Any]) -> Record | None: diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index 2a1bfe2..1c2400d 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -65,6 +65,8 @@ SPEC_OWNED = { "plane.generate": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set", "plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`", "plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free", + "plane.tasks.alerts": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free", + "plane.tasks.loop": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps", "plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`", "plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory", } diff --git a/src/openultrasast/plane/generate.py b/src/openultrasast/plane/generate.py index b755974..6f3bec3 100644 --- a/src/openultrasast/plane/generate.py +++ b/src/openultrasast/plane/generate.py @@ -15,8 +15,16 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi (``triage.json``, for the measurement); - ``tasks/.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two Workspaces and naming the files as env; -- ``runs/.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets, annotated with the - population and split (``openultrasast.io/population``/``split``: the files' stems) the memory rows carry. +- ``runs/.yaml``: the Run, per case ``facts -> verify a, verify b -> agree -> verify c -> final -> remember``, + with budgets, annotated with the population and split (``openultrasast.io/population``/``split``: the files' + stems) the memory rows carry. + +``--loop`` (harnessx-removal design section 6) appends the improvement loop: per case ``alerts`` (quick-mode rules +on the vulnerable and the fixed pin, which gets its Workspace ``-fixed``; ``case.json`` then also carries the +fix's new-side ranges) feeding ``remember``, and once ``memory-snapshot`` (the store's rows after the train-on-test +guard, written on the host by ``memory.seed``) -> ``loop-measure`` -> ``loop-propose`` -> ``loop-improve``, the last +two bound to this repository at the commit under test. All of them model-free with budget ``{usd: 0, calls: 0}``; +the gate's verdict is ``/loop-improve/gate.json``, and adopting an accepted ledger stays a maintainer commit. Each ``repo-facts`` Task carries ``openultrasast.io/memory-key`` (repository, pin, candidates digest, runner image digest): the key under which stored facts are reused instead of recomputed (``plane/memory.py``, ``seed``). @@ -57,12 +65,12 @@ import yaml from ..model.endpoint import price_of from .budget import cost_of from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests -from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, memory_key +from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SNAPSHOT_ANNOTATION, SPLIT_ANNOTATION, memory_key from .reconciler import _ax_name, _doc from .tasks.repo_facts import _digest from .tasks.verify import MAX_STEPS, units_of -__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"] +__all__ = ["CaseInputs", "Loop", "fix_ranges", "increment", "read_cases", "render", "repin_templates"] REFERENCE_USD_PER_CANDIDATE = 0.022 # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes REFERENCE_MODEL = "deepseek-flash" # verify_batched.py's default (DEFAULT_DETECTOR_MODEL) in the batched check @@ -76,7 +84,12 @@ CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content ATESPACE = "default" CASE_PATH = "/workspace/case" INPUTS_PATH = "/workspace/inputs" -TEMPLATES = ("repo-facts", "verify", "agree") +FIXED_PATH = "/workspace/fixed" +PROJECT_PATH = "/workspace/project" +TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop") +FIXED_RANGES_BASIS = "git diff -U0 , new side: the fix's lines at the fixed pin" +LOOP_MANIFEST = "benchmarks/manifests/java-spring-boot-vulnerable.toml" # the equality baseline's `improve` manifest +LOOP_CATALOG = "benchmarks/pairs/catalog.toml" Candidate = tuple[str, str, int] @@ -90,23 +103,40 @@ class CaseInputs: recorded_usd: float triage_usd: float = 0.0 ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None + fixed_ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None -def fix_ranges(repo: Path, vulnerable: str, fixed: str) -> dict[str, list[tuple[int, int]]]: +@dataclass(frozen=True) +class Loop: + """The improvement loop appended to a Run (``--loop``, design section 6): ``repo`` at ``commit`` is this + repository at the commit under test, where ``manifest`` and ``catalog`` (paths inside it) gate the proposals; + rows of ``populations`` never propose (the train-on-test guard).""" + + repo: str + commit: str + manifest: str = LOOP_MANIFEST + catalog: str = LOOP_CATALOG + populations: tuple[str, ...] = () + + +def fix_ranges(repo: Path, vulnerable: str, fixed: str, side: str = "old") -> dict[str, list[tuple[int, int]]]: """Changed line ranges per file on the vulnerable side of the fix: `evaluate.hunks(case, "old")`, line for - line (including its attribution of a hunk under ``--- /dev/null`` to the file before it).""" + line (including its attribution of a hunk under ``--- /dev/null`` to the file before it). ``side="new"`` is + the same on the fixed side (``+++ b/`` paths, the ``+`` spans): where a fixed-pin alert sits inside the fix.""" if not (repo / ".git").exists() and not (repo / "HEAD").is_file(): raise ValueError(f"no repository at {repo}: the fix ranges cannot be computed") diff = subprocess.run(["git", "-C", str(repo), "diff", "-U0", vulnerable, fixed], capture_output=True, text=True, check=True).stdout ranges: dict[str, list[tuple[int, int]]] = {} current = "" + header, group = ("--- a/", 1) if side == "old" else ("+++ b/", 3) for line in diff.splitlines(): - if line.startswith("--- a/"): + if line.startswith(header): current = line[6:] elif line.startswith("@@") and current: match = re.search(r"-(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?", line) if match: - first, length = int(match.group(1)), int(match.group(2)) if match.group(2) is not None else 1 + span = match.group(group + 1) + first, length = int(match.group(group)), int(span) if span is not None else 1 ranges.setdefault(current, []).append((first, first + max(length, 1) - 1)) if not ranges: raise ValueError(f"{repo}: the diff {vulnerable[:12]}..{fixed[:12]} changed no line on the vulnerable side") @@ -133,10 +163,11 @@ def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]: def read_cases( - validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None + validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None, *, fixed_side: bool = False ) -> list[CaseInputs]: """The set's cases in id order, each with its candidates, the triage-kept subset, the recorded cost and its - triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges.""" + triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges (and, with + ``fixed_side``, its new-side ranges, which the loop's `alerts` task marks fixed-pin alerts by).""" subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8")) records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]} cases: list[CaseInputs] = [] @@ -166,6 +197,11 @@ def read_cases( recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6), triage_usd=_triage_usd(rows), ranges=None if repos is None else fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"]), + fixed_ranges=( + fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"], side="new") + if repos is not None and fixed_side + else None + ), ) ) return cases @@ -188,6 +224,9 @@ def _case_record(item: CaseInputs) -> dict[str, Any]: if item.ranges is not None: record["ranges"] = {path: [list(span) for span in spans] for path, spans in item.ranges.items()} record["ranges_basis"] = RANGES_BASIS + if item.fixed_ranges is not None: + record["fixed_ranges"] = {path: [list(span) for span in spans] for path, spans in item.fixed_ranges.items()} + record["fixed_ranges_basis"] = FIXED_RANGES_BASIS record["sites_in_set"] = [s for s in sites if s in in_set] record["cost"] = { "candidates_before_triage": len(item.candidates), "recorded_usd": item.recorded_usd, @@ -213,11 +252,13 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]: return pinned, inputs -def _task(template: Task, name: str, env: Mapping[str, str], bindings: Sequence[tuple[str, str]]) -> dict[str, Any]: +def _task( + template: Task, name: str, env: Mapping[str, str], bindings: Sequence[tuple[str, str]], command: Sequence[str] = () +) -> dict[str, Any]: spec: dict[str, Any] = {} if template.image: spec["image"] = template.image - spec["command"] = list(template.command) + spec["command"] = [*template.command, *command] spec["env"] = [{"name": e.name, "value": e.value} for e in template.env] + [{"name": k, "value": v} for k, v in env.items()] if template.resources: kinds = {kind: values for kind, values in vars(template.resources).items() if values is not None} @@ -257,11 +298,91 @@ def _tiebreak_entries(item: CaseInputs) -> list[dict[str, Any]]: ] # fmt: skip +def _free() -> dict[str, int]: + return {"usd": 0, "calls": 0} # model-free: budget.py refuses any call, so a stray one fails loudly + + +def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]: + """(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the + repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) as input.""" + case_id = item.case["id"] + env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"} + task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)]) + inputs = {"facts": f"{case_id}-facts/facts.json", "facts_summary": f"{case_id}-facts/summary.json"} + for label in ("a", "b", "c"): + inputs.update({f"pass_{label}": f"{case_id}-v{label}/units.jsonl", f"summary_{label}": f"{case_id}-v{label}/summary.json"}) + inputs["agreed"] = f"{case_id}-final/agreed.json" + if alerts: + inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl" + entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()} + return task, entry + + +def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict, dict]: + """(fixed-pin Workspace, Task, Run entry) of the case's `alerts`: quick-mode rules on both pins.""" + case = item.case + git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}] + fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, ATESPACE) + fixed["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['fixed']}"} + bindings = [(_ax_name(case["id"], "vulnerable"), CASE_PATH), (fixed["metadata"]["name"], FIXED_PATH)] + bindings.append((_ax_name(case["id"], "inputs"), INPUTS_PATH)) + env = { + "OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo", "OUSAST_FIXED_DIR": f"{FIXED_PATH}/repo", + "OUSAST_INPUT_CASE": f"{INPUTS_PATH}/case.json", "OUSAST_VULNERABLE_PIN": str(case["vulnerable"]), + "OUSAST_FIXED_PIN": str(case["fixed"]), + } # fmt: skip + task = _task(templates["alerts"], f"alerts-{case['id']}", env, bindings) + entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl"], "budget": _free()} + entry["serialize"] = "engine" + return fixed, task, entry + + +def _loop(cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop) -> tuple[dict, list[dict], list[dict]]: + """(project Workspace, Tasks, Run entries) of the loop's singletons: snapshot -> measure -> propose -> improve.""" + git = [{"name": "repo", "repo": loop.repo, "dir": "repo", "depth": 1}] + project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, ATESPACE) + project["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={loop.commit}"} + bound = [(project["metadata"]["name"], PROJECT_PATH)] + env = { + "OUSAST_PROJECT_DIR": f"{PROJECT_PATH}/repo", "OUSAST_LOOP_MANIFEST": loop.manifest, "OUSAST_LOOP_CATALOG": loop.catalog, + "OUSAST_QUALIFY_POPULATIONS": json.dumps(sorted(loop.populations)), + } # fmt: skip + snapshot = _task(templates["loop"], "memory-snapshot", {}, [], ["snapshot"]) + guard = {"catalog": loop.catalog, "manifest": loop.manifest, "populations": sorted(loop.populations)} + snapshot["metadata"].setdefault("annotations", {})[SNAPSHOT_ANNOTATION] = json.dumps(guard, sort_keys=True, separators=(",", ":")) + tasks = [ + snapshot, _task(templates["loop"], "loop-measure", {}, [], ["measure"]), + _task(templates["loop"], "loop-propose", env, bound, ["propose"]), + _task(templates["loop"], "loop-improve", env, bound, ["improve"]), + ] # fmt: skip + remembered = {f"remember_{c.case['id']}": f"{c.case['id']}-remember/memory.jsonl" for c in cases} + entries = [ + {"name": "memory-snapshot", "task": "memory-snapshot", "outputs": ["rows.jsonl", "index.json"], "budget": _free()}, + {"name": "loop-measure", "task": "loop-measure", "inputs": {**remembered, "snapshot": "memory-snapshot/rows.jsonl"}, + "outputs": ["measure.json", "rows.jsonl"], "budget": _free()}, + {"name": "loop-propose", "task": "loop-propose", + "inputs": {"measure": "loop-measure/measure.json", "rows": "loop-measure/rows.jsonl", "index": "memory-snapshot/index.json"}, + "outputs": ["proposals.jsonl", "memory_proposals.jsonl", "signals.json"], "budget": _free()}, + {"name": "loop-improve", "task": "loop-improve", "inputs": {"proposals": "loop-propose/proposals.jsonl"}, + "outputs": ["gate.json", "journal.json"], "budget": _free(), "serialize": "engine"}, + ] # fmt: skip + return project, tasks, entries + + def render( - cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str, *, population: str = "", split: str = "" + cases: Sequence[CaseInputs], + templates: Mapping[str, Task], + run_name: str, + command: str, + *, + population: str = "", + split: str = "", + loop: Loop | None = None, ) -> dict[str, str]: """Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split`` - (the population file's and the set's stems) become the Run's annotations, which the memory rows carry.""" + (the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each + case ends in `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the + snapshot, `loop-measure`, `loop-propose` and `loop-improve` (design section 6), all model-free.""" header = f"# generated by `{command}`; do not edit, regenerate.\n" out: dict[str, str] = {} tasks: list[dict[str, Any]] = [] @@ -307,6 +428,20 @@ def render( tiebreak.extend(_tiebreak_entries(item)) tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state + for item in cases: # appended after the tie-break for the same reason + if loop is not None: + fixed, alerts_task, alerts_entry = _alerts(item, templates) + out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed]) + tasks.append(alerts_task) + entries.append(alerts_entry) + remember_task, remember_entry = _remember(item, templates, population, split, loop is not None) + tasks.append(remember_task) + entries.append(remember_entry) + if loop is not None: + project, loop_tasks, loop_entries = _loop(cases, templates, loop) + out[f"workspaces/{project['metadata']['name']}.yaml"] = _dump(header, [project]) + tasks.extend(loop_tasks) + entries.extend(loop_entries) out[f"tasks/{run_name}.yaml"] = _dump(header, tasks) annotations = {k: v for k, v in ((POPULATION_ANNOTATION, population), (SPLIT_ANNOTATION, split)) if v} metadata: dict[str, Any] = {"name": run_name, **({"annotations": annotations} if annotations else {})} @@ -349,14 +484,17 @@ def increment( command: str, repos: Path | None = None, runner_image: Path | None = None, + loop: Loop | None = None, ) -> list[Path]: """Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when - ``runner_image`` is given); returns the paths.""" + ``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths.""" + if loop is not None and repos is None: + raise ValueError("--loop needs --repos: the alerts task marks fixed-pin alerts by the fix's new-side ranges") if runner_image is not None: repin_templates(plane, runner_image) loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES]) - cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos) - files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem) + cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos, fixed_side=loop is not None) + files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop) written: list[Path] = [] for relative, text in files.items(): path = plane / relative diff --git a/src/openultrasast/plane/memory.py b/src/openultrasast/plane/memory.py index 6e15863..c620661 100644 --- a/src/openultrasast/plane/memory.py +++ b/src/openultrasast/plane/memory.py @@ -20,6 +20,10 @@ Fact reuse: a facts entry is valid for (repo, pin, candidates digest, runner ima writes as the Task annotation ``openultrasast.io/memory-key``. :func:`seed` runs before a Run: for every entry whose Task carries a key with stored facts and no state yet, it writes ``//facts.json`` and a ``done`` summary and marks the task done under the run lock (``reconciler.mark_done``), so the reconciler skips it. + +The loop's ``memory-snapshot`` (a Task annotated ``openultrasast.io/memory-snapshot`` with the train-on-test guard's +parameters) is seeded the same way: a sandboxed task cannot read this store, so :func:`seed` writes the store's rows +after the guard into ``/memory-snapshot/`` (:func:`.tasks.loop.write_snapshot`) and marks it done. """ from __future__ import annotations @@ -44,6 +48,7 @@ log = logging.getLogger(__name__) MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key" POPULATION_ANNOTATION = "openultrasast.io/population" SPLIT_ANNOTATION = "openultrasast.io/split" +SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"} KINDS = ("facts", "verdict", "unit_cost", "alert", "proposal_outcome") ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image") TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split") @@ -631,7 +636,8 @@ def ingest(run_dir: Path, store: MemoryStore | None = None) -> list[IngestResult def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root: Path | None = None) -> list[str]: """Reuse stored facts for a Run's tasks whose ``openultrasast.io/memory-key`` matches (Req 6.1): each such task without a directory or a state gets ``facts.json`` and a ``done`` summary and is marked done under the run - lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes.""" + lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes. + A ``memory-snapshot`` Task gets the store's rows after the guard its annotation names, marked done likewise.""" from . import reconciler run_spec, manifests = reconciler.load_run(Path(run_manifest)) @@ -640,10 +646,19 @@ def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root: state = (reconciler._read_json(base / "state.json") or {}).get("tasks") or {} seeded: list[str] = [] for entry in run_spec.tasks: - text = manifests.tasks[entry.task].metadata.annotations.get(MEMORY_KEY_ANNOTATION) - if not text or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending": + annotations = manifests.tasks[entry.task].metadata.annotations + text, snapshot = annotations.get(MEMORY_KEY_ANNOTATION), annotations.get(SNAPSHOT_ANNOTATION) + if not (text or snapshot) or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending": continue - key = parse_memory_key(text) + if snapshot: + from .tasks.loop import write_snapshot # lazy: the guard reads the pair catalog and the benchmark manifest + + info = write_snapshot(base / entry.name, store, json.loads(snapshot)) + reused = {"snapshot": store.describe(), **{k: info[k] for k in ("rows", "kept", "index_digest")}} + reconciler.mark_done(run_spec.metadata.name, entry.name, reused=reused, results_root=results_root) + seeded.append(entry.name) + continue + key = parse_memory_key(str(text)) where = {"candidates_digest": key["candidates"], "image": key["image"]} for record in sorted(store.rows(key["repo"], key["pin"], "facts", where), key=lambda r: str(r.row["run"]), reverse=True): data = store.get_facts(str(record.row["sha256"])) @@ -672,6 +687,7 @@ __all__ = [ "KINDS", "MEMORY_KEY_ANNOTATION", "POPULATION_ANNOTATION", + "SNAPSHOT_ANNOTATION", "SPLIT_ANNOTATION", "FileStore", "IngestResult", diff --git a/src/openultrasast/plane/tasks/alerts.py b/src/openultrasast/plane/tasks/alerts.py new file mode 100644 index 0000000..f9916a4 --- /dev/null +++ b/src/openultrasast/plane/tasks/alerts.py @@ -0,0 +1,123 @@ +"""`alerts` (harnessx-removal Requirement 6.3, design section 6): what the deterministic rules say about a case. + +No model, budget ``{usd: 0, calls: 0}``, one per case in a ``--loop`` Run, serialized like ``repo-facts`` (a full-tree +scan per pin). It runs the quick-mode scan -- ``preprocess_repository``, the entry-point reachability hints, +``rank_targets`` and ``quick_scan_findings`` over the image's bundled ruleset, enabled and shadow rules alike, exactly +what ``ousast scan --mode quick`` runs -- on the case's vulnerable pin and on its fixed pin, and writes one row per +(rule, file, line, pin) to ``alerts.jsonl``: + + rule_id, rule_status (enabled | shadow), path, line, function, pin_role (vulnerable | fixed), pin, in_fix_range + +``function`` is the enclosing declaration by the patterns ``repo-facts`` uses (so an alert joins the verify +candidates' ``path::function``; ```` outside any function); a language those patterns do not cover keeps the +scan's own reachability attribution. ``in_fix_range`` says whether the line lies inside the fix: the old-side ranges +of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fixed pin (null when ``case.json`` has +none). A fixed-pin alert inside the fix is a false alert on code the fix wrote: rule M1 of ``improve/memory.py`` +counts it. + +Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_WORKSPACE_DIR`` (vulnerable checkout), ``OUSAST_FIXED_DIR`` (fixed checkout), +``OUSAST_INPUT_CASE`` (``case.json``), ``OUSAST_VULNERABLE_PIN``, ``OUSAST_FIXED_PIN``. A pin whose checkout yields no +source file fails the task: an unread tree and a silent one are indistinguishable afterwards. Exit 0 done, 2 failed. +""" + +from __future__ import annotations + +import json +import os +import sys +import traceback +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from ...findings import quick_scan_findings +from ...mapping import analyze_entry_points, attach_reachability_hints +from ...preprocess import detect_language, preprocess_repository +from ...rank import rank_targets +from .repo_facts import DECLARATION, GLOBAL, enclosing + +ROLES = ("vulnerable", "fixed") +DIRS = ("OUSAST_WORKSPACE_DIR", "OUSAST_FIXED_DIR") +PINS = ("OUSAST_VULNERABLE_PIN", "OUSAST_FIXED_PIN") +Ranges = Mapping[str, Sequence[Sequence[int]]] + + +def _lines(path: Path) -> list[str]: + try: + return path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return [] + + +def in_range(ranges: Ranges | None, path: str, line: int | None) -> bool | None: + if ranges is None: + return None + return line is not None and any(int(lo) <= line <= int(hi) for lo, hi in ranges.get(path, [])) + + +def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, int]]: + """(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line.""" + _, targets = preprocess_repository(root) + targets = attach_reachability_hints(targets, analyze_entry_points(root, targets)) + findings = quick_scan_findings(root, targets, rank_targets(targets)) + read = {"files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file())} + cache: dict[str, list[str]] = {} + rows: dict[tuple[str, str, int | None], dict[str, Any]] = {} + for finding in findings: + rule_id = finding.finding_id.split(":", 1)[0] + lines = cache.setdefault(finding.path, _lines(root / finding.path)) + language = detect_language(root / finding.path) + if language in DECLARATION and finding.line and 0 < finding.line <= len(lines): + function = enclosing(lines, finding.line - 1, language) + else: + function = finding.function_name or GLOBAL + row = { + "rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line, "function": function, + "pin_role": role, "pin": pin, "in_fix_range": in_range(ranges, finding.path, finding.line), + } # fmt: skip + rows.setdefault((rule_id, finding.path, finding.line), row) + ordered = sorted(rows.values(), key=lambda r: (r["path"], r["line"] or 0, r["rule_id"])) + return ordered, read + + +def _summary(output_dir: Path, summary: Mapping[str, Any]) -> None: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(environ: Mapping[str, str] | None = None) -> int: + env = environ if environ is not None else os.environ + output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None + summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 2, "usd": 0, "calls": 0, "usage": {}, "model": None} + try: + if output is None: + raise ValueError("OUSAST_OUTPUT_DIR is not set") + output.mkdir(parents=True, exist_ok=True) + case = json.loads(Path(env["OUSAST_INPUT_CASE"]).read_text(encoding="utf-8")) if env.get("OUSAST_INPUT_CASE") else {} + ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")} + rows: list[dict[str, Any]] = [] + for role, directory, pin in zip(ROLES, DIRS, PINS, strict=True): + root = Path(env.get(directory) or "") + if not env.get(directory) or not root.is_dir(): + raise ValueError(f"{directory} is not a directory: {root}") + found, read = scan(root, role, str(env.get(pin) or ""), ranges[role]) + if read["files"] == 0: + raise ValueError(f"the {role} checkout {root} has no source file the scan reads: an unread tree is not a clean one") + summary.setdefault("read", {})[role] = read + summary.setdefault("alerts", {})[role] = len(found) + rows.extend(found) + summary["units_done"] += 1 + (output / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8") + summary["status"] = "done" + except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback + summary["reason"] = traceback.format_exc()[-2000:] + if output is not None: + _summary(output, summary) + print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read")}), file=sys.stderr) + return 0 if summary["status"] == "done" else 2 + + +__all__ = ["in_range", "main", "scan"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/openultrasast/plane/tasks/loop.py b/src/openultrasast/plane/tasks/loop.py new file mode 100644 index 0000000..3fe87e6 --- /dev/null +++ b/src/openultrasast/plane/tasks/loop.py @@ -0,0 +1,297 @@ +"""The improvement loop as plane tasks (harnessx-removal Requirement 6.3, design section 6). + +``command: ["loop", ]``; every step is model-free with budget ``{usd: 0, calls: 0}`` (``budget.py`` refuses any +call, so a stray one fails loudly), and each writes ``summary.json`` under the task contract. Records travel as +``{"key", "version", "row"}`` lines: the store object a row came from and its version (MinIO's version id, a file's +content hash), which a proposal's provenance cites (Req 6.4). + +- ``snapshot``: never runs in the sandbox, which cannot read the host's store. ``ousast plane run`` seeds it before + the Run (``memory.seed`` -> :func:`write_snapshot`): the store's rows after the train-on-test guard to + ``rows.jsonl``, and ``index.json`` (store, index digest, row counts, what the guard excluded). Run in a sandbox it + fails, naming the seed that did not happen. +- ``measure``: every case's ``remember`` rows (inputs ``remember_``) plus the snapshot -> ``measure.json``, + the increment's metrics for this run, the store and both (declared sites agreed, usd per candidate, dispute rates + per family, alerts per rule and pin) -- and ``rows.jsonl``, the union by row id that ``propose`` reads. +- ``propose``: ``improve/memory.py``'s rules M1/M2 over those rows after the guard (``OUSAST_QUALIFY_POPULATIONS``, the + gated manifest's cases, the pair catalog's holdout pairs), against the benchmark target's ledger and journal in + this repository at the commit under test (``OUSAST_PROJECT_DIR``) -> ``proposals.jsonl`` (edit + provenance), + ``memory_proposals.jsonl`` (the sidecar) and ``signals.json`` (M3, advisory). +- ``improve``: one ``evolve.run_round`` with those proposals -- the unchanged validator and gate, the pair catalog's + holdout clause included -- on copies of the target's ledger and journal -> ``gate.json`` (the ``RoundOutcome`` + and which edits came from memory), ``journal.json``, the sidecar, and ``rule_policy.json`` only when the round + was accepted; ``memory.jsonl`` holds the ``proposal_outcome`` rows the host ingests after the Run. Adopting an + accepted ledger stays a maintainer commit. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sys +import tempfile +import traceback +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from ..memory import REMEMBER_OUTPUT, MemoryStore, Record, validate_row + +INPUT_PREFIX = "OUSAST_INPUT_REMEMBER_" +CALIBRATION = Path(".openultrasast") / "calibration" # cli.CALIBRATION_DIR: where `improve` keeps a target's ledger +ADOPTION = "adoption is a maintainer commit: copy rule_policy.json to {ledger} (README, Rule-level loop)" + + +# --- records ------------------------------------------------------------------------------------------------------- + + +def _record_line(record: Record) -> str: + return json.dumps({"key": record.key, "version": record.version, "row": record.row}, sort_keys=True, separators=(",", ":")) + "\n" + + +def read_records(path: Path) -> list[Record]: + records: list[Record] = [] + for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if raw.strip(): + item = json.loads(raw) + records.append(Record(validate_row(item["row"], f"{path}:{number}"), str(item["key"]), item.get("version"))) + return records + + +def write_records(path: Path, records: Iterable[Record]) -> None: + path.write_text("".join(_record_line(r) for r in sorted(records, key=lambda r: str(r.row["id"]))), encoding="utf-8") + + +def _guard(params: Mapping[str, Any], root: Path) -> Any: + from ...benchmark import load_benchmark_manifest + from ...improve.memory import Guard, benchmark_cases + from ...pairs import load_pair_catalog, select_split + + manifest = load_benchmark_manifest(root / str(params["manifest"])) + holdout = select_split(load_pair_catalog(Path(str(params["catalog"]))), "holdout") + return Guard.build([str(p) for p in params.get("populations") or []], benchmark_cases(manifest), holdout) + + +def write_snapshot(directory: Path, store: MemoryStore, params: Mapping[str, Any], root: Path | None = None) -> dict[str, Any]: + """The store's rows after the guard ``params`` names (``catalog``, ``manifest``, ``populations``; paths relative to + ``root``, the working directory) into ``directory``: ``rows.jsonl``, ``index.json`` and a ``done`` summary.""" + from ...improve.memory import index_digest + + here = Path.cwd() + try: + os.chdir(root or here) # the pair catalog names its files relative to the repository root + guard = _guard(params, Path.cwd()) + finally: + os.chdir(here) + directory.mkdir(parents=True, exist_ok=False) # after the guard loaded: a failed load leaves no half-seeded task + records = store.rows() + kept, excluded = guard.apply(records) + write_records(directory / "rows.jsonl", kept) + info = { + "store": store.describe(), "index_entries": len(store.index()), "index_digest": index_digest(store), "rows": len(records), + "kept": len(kept), "excluded": excluded, "guard": dict(params), + } # fmt: skip + (directory / "index.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n", encoding="utf-8") + summary = {"status": "done", "units_done": 1, "units_total": 1, "usd": None, "calls": 0, "usage": {}, "model": None, "reused": info} + (directory / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return info + + +# --- measure ------------------------------------------------------------------------------------------------------- + + +def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """The increment's metrics over verdict, unit-cost and alert rows (a disputed final is neither agreed nor not).""" + verdicts = [r for r in rows if r["kind"] == "verdict"] + finals = {f: sum(1 for v in verdicts if v.get("final") == f) for f in ("agreed", "rejected", "disputed")} + costs = [r.get("usd") for r in rows if r["kind"] == "unit_cost"] + usd = None if any(c is None for c in costs) else round(sum(float(c or 0) for c in costs), 6) + families: dict[str, list[int]] = {} + for v in verdicts: + tally = families.setdefault(str(v.get("family")), [0, 0]) + tally[0] += 1 + tally[1] += v.get("final") == "disputed" + alerts: dict[str, dict[str, int]] = {} + for a in (r for r in rows if r["kind"] == "alert"): + role = "fixed" if a.get("pin_role") == "fixed" else "vulnerable" + tally_a = alerts.setdefault(str(a.get("rule_id")), {"vulnerable": 0, "fixed": 0, "fixed_in_fix_range": 0}) + tally_a[role] += 1 + tally_a["fixed_in_fix_range"] += role == "fixed" and a.get("in_fix_range") is True + return { + "cases": len({(v["repo"], v["pin"]) for v in verdicts}), + "candidates": len(verdicts), + **finals, + "tiebreaks": sum(1 for v in verdicts if v.get("tiebreak")), + "declared_sites_matched": sum(1 for v in verdicts if v.get("site_match") is True), + "declared_sites_agreed": sum(1 for v in verdicts if v.get("site_match") is True and v.get("final") == "agreed"), + "usd": usd, + "usd_per_candidate": None if usd is None or not verdicts else round(usd / len(verdicts), 6), + "dispute_rate": {f: {"candidates": n, "disputed": d, "rate": round(d / n, 4)} for f, (n, d) in sorted(families.items())}, + "alerts": dict(sorted(alerts.items())), + } + + +def measure(env: Mapping[str, str], output: Path) -> dict[str, Any]: + run = env.get("OUSAST_RUN") or "unknown" + current: list[Record] = [] + read: dict[str, int] = {} + for name in sorted(k for k in env if k.startswith(INPUT_PREFIX)): + path = Path(env[name]) + data = path.read_bytes() + read[name.removeprefix(INPUT_PREFIX).lower()] = len(data) + version = hashlib.sha256(data).hexdigest() + for number, raw in enumerate(data.decode("utf-8").splitlines(), 1): + if raw.strip(): + row = validate_row(json.loads(raw), f"{path}:{number}") + current.append(Record(row, f"runs/{run}/{row['task']}/{REMEMBER_OUTPUT}", version)) + if not current: + raise ValueError(f"read no remember rows from {len(read)} inputs ({read}): an unread run is not an empty one") + snapshot = read_records(Path(env["OUSAST_INPUT_SNAPSHOT"])) if env.get("OUSAST_INPUT_SNAPSHOT") else [] + merged = {str(r.row["id"]): r for r in snapshot} + merged.update({str(r.row["id"]): r for r in current}) # this run's row wins over its earlier ingest + write_records(output / "rows.jsonl", merged.values()) + report = { + "run": run, "inputs_bytes": read, "rows": {"run": len(current), "snapshot": len(snapshot), "union": len(merged)}, + "run_metrics": metrics([r.row for r in current]), "store_metrics": metrics([r.row for r in snapshot]), + "union_metrics": metrics([r.row for r in merged.values()]), + } # fmt: skip + (output / "measure.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return {"rows": report["rows"]} + + +# --- propose and improve ------------------------------------------------------------------------------------------- + + +def _project(env: Mapping[str, str]) -> tuple[Any, Path, Path, Path]: + """(manifest, target, ledger, journal) in the repository at the commit under test; the working directory + becomes the repository root, where the pair catalog's relative paths resolve.""" + from ...benchmark import load_benchmark_manifest, resolve_benchmark_source + + project = Path(env.get("OUSAST_PROJECT_DIR") or "") + if not env.get("OUSAST_PROJECT_DIR") or not (project / str(env.get("OUSAST_LOOP_MANIFEST"))).is_file(): + raise ValueError(f"OUSAST_PROJECT_DIR {project} holds no benchmark manifest {env.get('OUSAST_LOOP_MANIFEST')}") + os.chdir(project) + manifest_path = Path(str(env["OUSAST_LOOP_MANIFEST"])) + manifest = load_benchmark_manifest(manifest_path) + target = resolve_benchmark_source(manifest_path, manifest) + ledger = target / CALIBRATION / "rule_policy.json" + return manifest, target, ledger, ledger.with_name("improve_journal.json") + + +def _params(env: Mapping[str, str]) -> dict[str, Any]: + populations = json.loads(env.get("OUSAST_QUALIFY_POPULATIONS") or "[]") + return {"catalog": env.get("OUSAST_LOOP_CATALOG"), "manifest": env.get("OUSAST_LOOP_MANIFEST"), "populations": populations} + + +def propose(env: Mapping[str, str], output: Path) -> dict[str, Any]: + from ...improve.journal import load_journal, next_round_index + from ...improve.memory import disputed_families, propose_from_memory, write_sidecar, write_signals + from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset, read_rule_ledger + + records = read_records(Path(env["OUSAST_INPUT_ROWS"])) + index = json.loads(Path(env["OUSAST_INPUT_INDEX"]).read_text(encoding="utf-8")) + _, _, ledger, journal = _project(env) + guard = _guard(_params(env), Path.cwd()) + rules = load_ruleset(DEFAULT_RULESET_DIR, ledger if ledger.is_file() else None) + rounds = load_journal(journal) + found = propose_from_memory( + records, {r.rule_id: r for r in rules}, read_rule_ledger(ledger), rounds, excluded=guard, index_digest=str(index["index_digest"]) + ) + lines = [json.dumps({"edit": asdict(p.edit), "provenance": p.provenance}, sort_keys=True) + "\n" for p in found] + (output / "proposals.jsonl").write_text("".join(lines), encoding="utf-8") + sidecar = output / "journal.json" + write_sidecar(sidecar, next_round_index(rounds), found) + sidecar.with_name("memory_proposals.jsonl").touch() # present when nothing was proposed, as a declared output + write_signals(output, disputed_families(records, guard)) + kept, excluded = guard.apply(records) + return {"proposals": len(found), "rows": len(records), "kept": len(kept), "excluded": excluded["rows"]} + + +def improve(env: Mapping[str, str], output: Path) -> dict[str, Any]: + from ...gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR + from ...improve.evolve import run_round + from ...improve.memory import MemoryProposal, outcome_of, outcome_rows, proposal_id_of + from ...improve.validator import RuleStatusEdit + from ...pairs import load_pair_catalog, select_split + from ...policy import load_policy + from ...ruleset import DEFAULT_RULESET_DIR + + path = Path(env["OUSAST_INPUT_PROPOSALS"]) + items = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + proposals = [MemoryProposal(RuleStatusEdit(**item["edit"]), item["provenance"]) for item in items] + manifest, target, ledger, journal = _project(env) + pairs = select_split(load_pair_catalog(Path(str(env["OUSAST_LOOP_CATALOG"]))), "holdout") + with tempfile.TemporaryDirectory(prefix="ousast-loop-") as scratch: + work_ledger = Path(scratch) / "rule_policy.json" + if ledger.is_file(): + shutil.copyfile(ledger, work_ledger) + if journal.is_file(): + shutil.copyfile(journal, output / "journal.json") + outcome = run_round( + target, manifest, ledger_path=work_ledger, journal_path=output / "journal.json", ruleset_dir=DEFAULT_RULESET_DIR, + policy=load_policy(), recall_floor=RECALL_FLOOR, fp_ceiling=FALSE_POSITIVE_CEILING, pair_cases=pairs, proposals=proposals, + ) # fmt: skip + if outcome.accepted and work_ledger.is_file(): + shutil.copyfile(work_ledger, output / "rule_policy.json") + (output / "journal.json").touch() + (output / "memory_proposals.jsonl").touch() + memory_edits = [e.key() for e in outcome.edits if proposal_id_of(e)] + verdict = { + "accepted": outcome.accepted, "reason": outcome.reason, "round": outcome.round, "proposals": len(proposals), + "memory_edits": memory_edits, "benchmark_edits": [e.key() for e in outcome.edits if not proposal_id_of(e)], + "manifest": env.get("OUSAST_LOOP_MANIFEST"), "outcome": asdict(outcome), + "adoption": ADOPTION.format(ledger=ledger.as_posix()) if outcome.accepted else None, + } # fmt: skip + (output / "gate.json").write_text(json.dumps(verdict, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8") + status = outcome_of(outcome.reason, outcome.accepted) + by_id = {p.proposal_id: p for p in proposals} + gate_run = f"plane:{env.get('OUSAST_RUN') or 'unknown'}" + rows = outcome_rows(outcome.edits, status, outcome.round, outcome.reason, by_id, gate_run) if status else [] + (output / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8") + return {"accepted": outcome.accepted, "reason": outcome.reason, "edits": len(outcome.edits), "memory_edits": len(memory_edits)} + + +def snapshot(env: Mapping[str, str], output: Path) -> dict[str, Any]: + raise RuntimeError( + "memory-snapshot ran in the sandbox: `ousast plane run` writes it on the host from the memory store before the Run " + "starts (memory.seed), so the seed did not happen; rerun through `ousast plane run`" + ) + + +STEPS: dict[str, Callable[[Mapping[str, str], Path], dict[str, Any]]] = { + "snapshot": snapshot, "measure": measure, "propose": propose, "improve": improve, +} # fmt: skip + + +def main(argv: Sequence[str] | None = None, environ: Mapping[str, str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + env = environ if environ is not None else os.environ + output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None + summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None} + here = Path.cwd() + try: + if output is None: + raise ValueError("OUSAST_OUTPUT_DIR is not set") + if len(args) != 1 or args[0] not in STEPS: + raise ValueError(f"expected one step of {sorted(STEPS)}, got {args}") + output.mkdir(parents=True, exist_ok=True) + summary.update(step=args[0], **STEPS[args[0]](env, output)) + summary.update(status="done", units_done=1) + except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback + summary["reason"] = traceback.format_exc()[-2000:] + finally: + os.chdir(here) + if output is not None: + (output / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8") + shown = {k: summary.get(k) for k in ("status", "step", "proposals", "accepted", "reason", "rows")} + print(json.dumps(shown, default=str), file=sys.stderr) + return 0 if summary["status"] == "done" else 2 + + +__all__ = ["main", "measure", "metrics", "propose", "improve", "read_records", "write_records", "write_snapshot"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/openultrasast/plane/tasks/remember.py b/src/openultrasast/plane/tasks/remember.py index 18b5193..dd66d8e 100644 --- a/src/openultrasast/plane/tasks/remember.py +++ b/src/openultrasast/plane/tasks/remember.py @@ -9,7 +9,8 @@ artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`, `population`, `split` and `image` (the runner image digest the case's tasks ran on). -Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS`, +Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS` (and `OUSAST_INPUT_FACTS_SUMMARY`, +whose reused or recorded `image` the facts row carries; without one named, the row is omitted), `OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes' `summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from @@ -40,6 +41,7 @@ from .repo_facts import _digest PASSES = ("a", "b", "c") UNKNOWN = "unknown" +ALERT_FIELDS = ("rule_id", "rule_status", "path", "line", "function", "pin_role") @dataclass(frozen=True) @@ -108,8 +110,13 @@ def rows_for( } # fmt: skip rows.append(ctx.row("unit_cost", f"{label}:{unit.get('path')}", **fields)) for alert in alerts: - fields = {k: alert.get(k) for k in ("rule_id", "rule_status", "path", "line", "function", "pin_role")} - rows.append(ctx.row("alert", f"{fields['rule_id']}:{fields['path']}:{fields['line']}", **fields)) + fields = {k: alert.get(k) for k in ALERT_FIELDS} + if "in_fix_range" in alert: # the loop's `alerts` task marks it; improve/memory.py's M1 reads it + fields["in_fix_range"] = alert["in_fix_range"] + if alert.get("pin"): # a fixed-pin alert is a row of the pin it was raised on, not of the case's vulnerable pin + fields["pin"] = str(alert["pin"]) + subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "") + rows.append(ctx.row("alert", subject, **fields)) return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows] @@ -202,6 +209,19 @@ def _path(env: Mapping[str, str], name: str) -> Path | None: return Path(env[name]) if env.get(name) else None +def _facts_image(env: Mapping[str, str], own: str) -> str | None: + """The image the case's facts were computed on: without `OUSAST_INPUT_FACTS_SUMMARY` this task's own; with it, + the reused entry's image or the image the facts task recorded, else None (unknown).""" + path = _path(env, "OUSAST_INPUT_FACTS_SUMMARY") + if path is None: + return own + summary = _json(path) or {} + reused = summary.get("reused") + reused = reused if isinstance(reused, dict) else {} + image = reused.get("image") or summary.get("image") + return image_digest(str(image)) if image else None + + def _units(path: Path | None) -> list[dict[str, Any]]: return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path) @@ -226,7 +246,13 @@ def main(environ: Mapping[str, str] | None = None) -> int: passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES} models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES} agreed = _json(_path(env, "OUSAST_INPUT_AGREED")) - rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS"))) + rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS"))) + facts_image = _facts_image(env, ctx.image) + if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on + facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split) + rows = rows_for(facts_ctx, facts=facts, passes={}, models={}, agreed=None) + rows + elif facts is not None: + summary["facts_row"] = "omitted: the facts task's summary names no image, and a reuse key must not guess it" (output_dir / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8") if facts is not None: (output_dir / "facts.json").write_bytes(facts) diff --git a/src/openultrasast/plane/tasks/repo_facts.py b/src/openultrasast/plane/tasks/repo_facts.py index 74fe397..c758054 100644 --- a/src/openultrasast/plane/tasks/repo_facts.py +++ b/src/openultrasast/plane/tasks/repo_facts.py @@ -23,6 +23,8 @@ from collections.abc import Iterable, Mapping, Sequence from pathlib import Path from typing import Any +import yaml + from ...model.layout import is_test_path, is_vendored, layout_facts from ...preprocess import IGNORED_DIRS, detect_language @@ -259,12 +261,13 @@ def assemble_facts( } -def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | None = None) -> dict[str, Any]: +def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | None = None, image: str | None = None) -> dict[str, Any]: """Compute the facts for ``workspace`` into ``output_dir``; returns the written summary. ``candidates`` rows are ``{"path", "function"}`` (workspace-relative POSIX path). ``None`` means every function every product file defines. A `units.jsonl` row from an earlier run over the same candidate set - is reused, not recomputed. + is reused, not recomputed. ``image`` (the runner image, from `AX_TASK_YAML`) goes into the summary: the + `remember` task keys the facts row by the image the facts were computed on. """ output_dir.mkdir(parents=True, exist_ok=True) units_path = output_dir / "units.jsonl" @@ -307,7 +310,8 @@ def run(workspace: Path, output_dir: Path, candidates: list[dict[str, Any]] | No raise TaskFailed(str(exc)) from exc facts = assemble_facts(rows, wanted) _write_json(output_dir / "facts.json", facts) - return write_summary(output_dir, status="done", units_done=len(rows), units_total=len(files)) + extra = {"image": image} if image else {} + return write_summary(output_dir, status="done", units_done=len(rows), units_total=len(files), **extra) def _load_candidates(path: str | None) -> list[dict[str, Any]] | None: @@ -332,7 +336,9 @@ def main(environ: Mapping[str, str] | None = None) -> int: workspace = Path(env.get("OUSAST_WORKSPACE_DIR") or "") if not env.get("OUSAST_WORKSPACE_DIR") or not workspace.is_dir(): raise ValueError(f"OUSAST_WORKSPACE_DIR is not a directory: {workspace}") - summary = run(workspace, output_dir, _load_candidates(env.get("OUSAST_INPUT_CANDIDATES"))) + task = yaml.safe_load(env.get("AX_TASK_YAML") or "{}") + image = str((task.get("spec") or {}).get("image") or "") if isinstance(task, dict) else "" + summary = run(workspace, output_dir, _load_candidates(env.get("OUSAST_INPUT_CANDIDATES")), image or None) except TaskFailed as exc: print(f"repo-facts failed: {exc}", file=sys.stderr) return 2 diff --git a/tests/test_plane_increment.py b/tests/test_plane_increment.py index fec33f6..665d660 100644 --- a/tests/test_plane_increment.py +++ b/tests/test_plane_increment.py @@ -56,7 +56,7 @@ def test_model_prices_are_the_endpoint_prices() -> None: def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None: run, manifests = reconciler.load_run(RUN) - assert len(run.tasks) == 90 + assert len(run.tasks) == 105 # per case: facts, va, vb, agree, then vc, final, then remember (harnessx-removal task 4) population = {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]} ax_names: set[str] = set() for task in manifests.tasks.values(): @@ -96,7 +96,7 @@ def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical( first = text.split(" - name: budibase-mongo-template-nosqli-vc\n", 1)[0] assert hashlib.sha256(first.encode()).hexdigest() == FIRST_RUN_ENTRIES run, manifests = reconciler.load_run(RUN) - old, new = run.tasks[:60], run.tasks[60:] + old, new, remembered = run.tasks[:60], run.tasks[60:90], run.tasks[90:] assert [e.name.rsplit("-", 1)[1] for e in old] == ["facts", "va", "vb", "agree"] * 15 assert [e.name.rsplit("-", 1)[1] for e in new] == ["vc", "final"] * 15 for vc, final in zip(new[::2], new[1::2], strict=True): @@ -112,6 +112,12 @@ def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical( for entry in (vc, final): name = reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT) assert len(name) <= reconciler.TASK_NAME_LIMIT + for entry in remembered: # harnessx-removal task 4: appended after the tie-break, model-free + case = entry.name.removesuffix("-remember") + assert entry.inputs["agreed"] == f"{case}-final/agreed.json" and entry.inputs["pass_c"] == f"{case}-vc/units.jsonl" + assert entry.budget is not None and (entry.budget.usd, entry.budget.calls) == (0, 0) + assert reconciler.MODEL_ANNOTATION not in manifests.tasks[entry.task].metadata.annotations + assert len(reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT)) <= reconciler.TASK_NAME_LIMIT def test_inputs_carry_the_validation_set() -> None: diff --git a/tests/test_plane_loop.py b/tests/test_plane_loop.py new file mode 100644 index 0000000..0fdadd5 --- /dev/null +++ b/tests/test_plane_loop.py @@ -0,0 +1,255 @@ +"""The improvement loop as a plane Run (harnessx-removal Req 6.3, design section 6). + +The whole graph runs through the reconciler and the fake ax of ``test_plane_reconciler``, whose ``exec`` mode runs the +real task modules on the inputs the receiver serves: a recorded case's chain (validation-46's lmdeploy case from +``tests/fixtures/plane-remember``, already done) -> ``alerts`` -> ``remember`` -> ``loop-measure`` -> ``loop-propose`` +-> ``loop-improve``, with ``memory-snapshot`` seeded on the host from a crafted store. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import pytest +import yaml +from test_plane_reconciler import Fake, fake # noqa: F401 -- `fake` is the fixture + +import openultrasast +from openultrasast import cli +from openultrasast.plane import reconciler +from openultrasast.plane.generate import TEMPLATES, CaseInputs, Loop, fix_ranges, render +from openultrasast.plane.manifests import load_manifests +from openultrasast.plane.memory import SNAPSHOT_ANNOTATION, FileStore, row_id +from openultrasast.plane.tasks import alerts + +FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46" +PLANE = Path(__file__).resolve().parents[1] / "plane" +SRC = str(Path(openultrasast.__file__).resolve().parents[1]) +CASE = "lmdeploy-media-url-ssrf" +VULNERABLE = "71648d239c978452720a6d6692fe914efc8a7b94" +FIXED = "ab" * 20 +OTHER = "12" * 20 +EVAL = "python-unsafe-eval" # a bundled, enabled rule +CHAIN = ("facts", "va", "vb", "agree", "vc", "final") +LOOP_TASKS = ("memory-snapshot", "loop-measure", "loop-propose", "loop-improve") + + +def _case(**ranges: Any) -> CaseInputs: + record = { + "id": CASE, "repo": "https://example.com/reserved/lmdeploy", "family": "untrusted_destination", "vulnerable": VULNERABLE, + "fixed": FIXED, "sites": ["lmdeploy/vl/media/connection.py::_load_http_url"], + } # fmt: skip + candidate = ("lmdeploy/vl/media/connection.py", "_load_http_url", 40) + return CaseInputs(record, (candidate,), (candidate,), {}, 0.02, **ranges) + + +def _templates() -> dict: + return load_manifests([PLANE / "tasks" / f"{name}.yaml" for name in TEMPLATES]).tasks + + +LOOP = Loop(repo="https://example.invalid/openultrasast.git", commit="c" * 40, manifest="bench.toml", catalog="catalog.toml") + + +def _write(plane: Path, files: dict[str, str]) -> Path: + shutil.copytree(PLANE / "models", plane / "models") + for relative, text in files.items(): + (plane / relative).parent.mkdir(parents=True, exist_ok=True) + (plane / relative).write_text(text) + return plane / "runs" / "validation-46.yaml" + + +# --- alerts -------------------------------------------------------------------------------------------------------- + + +def test_alerts_attribute_functions_and_mark_the_fix_on_both_pins(tmp_path: Path) -> None: + vulnerable, fixed = tmp_path / "v", tmp_path / "f" + (vulnerable / "pkg").mkdir(parents=True) + (fixed / "pkg").mkdir(parents=True) + (vulnerable / "pkg" / "run.py").write_text("import os\n\ndef handle(x):\n y = x\n return eval(y)\n") + (fixed / "pkg" / "run.py").write_text("import os\n\ndef handle(x):\n return eval(x, {})\n\nprint(eval('1'))\n") + case = tmp_path / "case.json" + case.write_text(json.dumps({"ranges": {"pkg/run.py": [[4, 5]]}, "fixed_ranges": {"pkg/run.py": [[4, 4]]}})) + out = tmp_path / "out" + env = { + "OUSAST_OUTPUT_DIR": str(out), "OUSAST_WORKSPACE_DIR": str(vulnerable), "OUSAST_FIXED_DIR": str(fixed), + "OUSAST_INPUT_CASE": str(case), "OUSAST_VULNERABLE_PIN": VULNERABLE, "OUSAST_FIXED_PIN": FIXED, + } # fmt: skip + assert alerts.main(env) == 0 + rows = [json.loads(line) for line in (out / "alerts.jsonl").read_text().splitlines()] + got = [(r["rule_id"], r["pin_role"], r["pin"], r["line"], r["function"], r["in_fix_range"]) for r in rows if r["rule_id"] == EVAL] + assert got == [ + (EVAL, "vulnerable", VULNERABLE, 5, "handle", True), + (EVAL, "fixed", FIXED, 4, "handle", True), + (EVAL, "fixed", FIXED, 6, "", False), + ] + summary = json.loads((out / "summary.json").read_text()) + assert (summary["status"], summary["units_done"], summary["usd"], summary["calls"], summary["model"]) == ("done", 2, 0, 0, None) + assert summary["read"]["vulnerable"]["files"] == 1 and summary["read"]["vulnerable"]["bytes"] > 0, "the scan read its tree" + assert alerts.in_range(None, "pkg/run.py", 4) is None, "no ranges: unknown, not outside" + + empty = tmp_path / "empty" + empty.mkdir() + assert alerts.main({**env, "OUSAST_FIXED_DIR": str(empty)}) == 2 + assert "an unread tree is not a clean one" in json.loads((out / "summary.json").read_text())["reason"] + + +def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None: + repo = tmp_path / "r" + repo.mkdir() + git = ["git", "-C", str(repo), "-c", "user.email=t@t", "-c", "user.name=t"] + subprocess.run([*git, "init", "-q"], check=True) + (repo / "a.py").write_text("a\nb\nc\n") + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-qm", "v"], check=True) + old = subprocess.run([*git, "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip() + (repo / "a.py").write_text("a\nB1\nB2\nB3\nc\n") + subprocess.run([*git, "commit", "-qam", "f"], check=True) + new = subprocess.run([*git, "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip() + assert fix_ranges(repo, old, new) == {"a.py": [(2, 2)]} + assert fix_ranges(repo, old, new, side="new") == {"a.py": [(2, 4)]} + + +# --- the generator ------------------------------------------------------------------------------------------------- + + +def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path) -> None: + plain = render([_case()], _templates(), "validation-46", "t", population="population-v2", split="validation") + run, _ = reconciler.load_run(_write(tmp_path / "plain", plain)) + assert [e.name.removeprefix(f"{CASE}-") for e in run.tasks] == [*CHAIN[:4], "vc", "final", "remember"] + remember = run.task(f"{CASE}-remember") + assert remember.inputs["agreed"] == f"{CASE}-final/agreed.json" and "alerts" not in remember.inputs + assert set(remember.producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final")} + + files = render([_case(fixed_ranges={"x.py": [(1, 2)]})], _templates(), "validation-46", "t", population="p", split="s", loop=LOOP) + run, manifests = reconciler.load_run(_write(tmp_path / "loop", files)) + names = [e.name for e in run.tasks] + assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS] + assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl" + measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"} + assert run.task("loop-measure").inputs == measure_inputs + assert run.task("loop-improve").producers == ("loop-propose",) + assert run.task("loop-propose").producers == ("loop-measure", "memory-snapshot") + assert run.task(f"{CASE}-alerts").serialize == run.task("loop-improve").serialize == "engine" + fixed = manifests.workspaces[f"{CASE}-fixed"] + assert fixed.pins == {"repo": FIXED} and manifests.workspaces["openultrasast-cccccccccccc"].pins == {"repo": "c" * 40} + for entry in run.tasks[6:]: + task = manifests.tasks[entry.task] + assert entry.budget is not None and (entry.budget.usd, entry.budget.calls) == (0, 0), entry.name + assert reconciler.MODEL_ANNOTATION not in task.metadata.annotations, f"{entry.name} binds no Model" + docs = reconciler.render_task(run, entry, manifests, "http://127.0.0.1:1/") + env = {e["name"]: e["value"] for e in docs[-1]["spec"]["env"]} + assert "OUSAST_MODEL" not in env and float(env["OUSAST_BUDGET_USD"]) == 0 and env["OUSAST_BUDGET_CALLS"] == "0" + guard = json.loads(manifests.tasks["memory-snapshot"].metadata.annotations[SNAPSHOT_ANNOTATION]) + assert guard == {"catalog": "catalog.toml", "manifest": "bench.toml", "populations": []} + inputs = yaml.safe_load((tmp_path / "loop" / "workspaces" / f"{CASE}-inputs.yaml").read_text()) + case_json = json.loads(next(f["content"] for f in inputs["spec"]["files"] if f["path"] == "case.json")) + assert case_json["fixed_ranges"] == {"x.py": [[1, 2]]} + + +# --- the whole loop through the fake ax ---------------------------------------------------------------------------- + + +def _project(root: Path) -> Path: + """This repository at the commit under test, cut down: a benchmark whose one expected finding the eval rule + makes, and an empty pair catalog.""" + (root / "target").mkdir(parents=True) + (root / "target" / "app.py").write_text("def f(x):\n return eval(x)\n") + (root / "bench.toml").write_text( + 'name = "t"\nlanguage = "python_web"\n\n[source]\npath = "target"\n\n[[expected]]\ncwe = "CWE-95"\n' + f'class = "code injection"\npath = "app.py"\nline = 2\nrule_id = "{EVAL}"\nsink = "eval"\nevidence = "eval"\n' + ) + (root / "catalog.toml").write_text("") + (root / "benchmarks" / "pairs").mkdir(parents=True) # the catalog loader's mechanism vocabulary, by relative path + shutil.copy(PLANE.parent / "benchmarks" / "pairs" / "mechanisms.toml", root / "benchmarks" / "pairs") + return root + + +def _older_run(store: FileStore) -> None: + """Two false alerts of the eval rule at rejected candidates of another repository, from an earlier run.""" + rows = [] + for fn in ("f1", "f2"): + base = {"repo": "example.com/other/app", "pin": OTHER, "run": "older", "task": "app-remember", "population": "p", "split": "s"} + base["image"] = "sha256:" + "0" * 64 + cand = f"app.py::{fn}" + rows.append({**base, "id": row_id("verdict", "older", "app-remember", cand), "kind": "verdict", "candidate": cand, + "family": "code_injection", "final": "rejected", "site_match": False}) # fmt: skip + rows.append({**base, "id": row_id("alert", "older", "app-remember", fn), "kind": "alert", "rule_id": EVAL, + "rule_status": "enabled", "path": "app.py", "line": 3, "function": fn, "pin_role": "vulnerable"}) # fmt: skip + store.ingest_rows("older", "app-remember", rows) + + +def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_costs_recall( + fake: Fake, # noqa: F811 + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project = _project(tmp_path / "project") + monkeypatch.chdir(project) # `ousast plane run` from the repository root: the snapshot's guard paths resolve here + store = FileStore(tmp_path / "memory") + _older_run(store) + monkeypatch.setenv("OUSAST_MEMORY", f"file://{store.root}") + vulnerable, fixed = tmp_path / "checkout-v", tmp_path / "checkout-f" + for root, body in ((vulnerable, "def load(x):\n return x\n"), (fixed, "def load(x):\n return eval(x)\n")): + (root / "lmdeploy" / "vl" / "media").mkdir(parents=True) + (root / "lmdeploy" / "vl" / "media" / "other.py").write_text(body) + case = _case(ranges={"lmdeploy/vl/media/other.py": [(1, 2)]}, fixed_ranges={"lmdeploy/vl/media/other.py": [(1, 2)]}) + files = render([case], _templates(), "validation-46", "t", population="population-v2", split="validation", loop=LOOP) + manifest = _write(tmp_path / "plane", files) + inputs = yaml.safe_load((tmp_path / "plane" / "workspaces" / f"{CASE}-inputs.yaml").read_text()) + (tmp_path / "case.json").write_text(next(f["content"] for f in inputs["spec"]["files"] if f["path"] == "case.json")) + + base = reconciler.run_dir("validation-46") # the recorded chain, done: the loop reuses it and runs only itself + shutil.copytree(FIXTURE, base) + state = {"run": "validation-46", "tasks": {f"{CASE}-{s}": {"status": "done"} for s in CHAIN}} + (base / "state.json").write_text(json.dumps(state)) + + project_env = {"PYTHONPATH": SRC, "OUSAST_PROJECT_DIR": str(project)} + fake.script({ + f"{CASE}-alerts": {"exec": {"env": {"PYTHONPATH": SRC, "OUSAST_WORKSPACE_DIR": str(vulnerable), "OUSAST_FIXED_DIR": str(fixed), + "OUSAST_INPUT_CASE": str(tmp_path / "case.json")}}}, + f"{CASE}-remember": {"exec": {"env": {"PYTHONPATH": SRC}}}, + "loop-measure": {"exec": {"env": {"PYTHONPATH": SRC}}}, + "loop-propose": {"exec": {"env": project_env}}, + "loop-improve": {"exec": {"env": project_env}}, + }) # fmt: skip + + assert cli.main(["plane", "run", str(manifest), "--ax", str(fake.ax)]) == 0, (fake.root / "ax.log").read_text()[-3000:] + + statuses = {name: info["status"] for name, info in json.loads((base / "state.json").read_text())["tasks"].items()} + assert set(statuses.values()) == {"done"} and len(statuses) == 12 + assert json.loads((base / "memory-snapshot" / "index.json").read_text())["kept"] == 4, "seeded on the host, not run" + delivered = [name for _, name in fake.events("delivered")] + assert delivered == [f"{CASE}-alerts", f"{CASE}-remember", "loop-measure", "loop-propose", "loop-improve"] + + measure = json.loads((base / "loop-measure" / "measure.json").read_text()) + assert measure["run_metrics"]["declared_sites_agreed"] == 1 and measure["run_metrics"]["candidates"] == 1 + assert measure["run_metrics"]["alerts"] == {EVAL: {"vulnerable": 0, "fixed": 1, "fixed_in_fix_range": 1}} + assert measure["union_metrics"]["rejected"] == 2 and measure["rows"]["snapshot"] == 4 + + [proposal] = [json.loads(line) for line in (base / "loop-propose" / "proposals.jsonl").read_text().splitlines()] + assert proposal["edit"]["rule_id"] == EVAL and (proposal["edit"]["from_status"], proposal["edit"]["to_status"]) == ("enabled", "shadow") + assert proposal["provenance"]["counts"] == {"false_alerts": 3, "repos": 2, "agreed_hits": 0} + cited = {(e["kind"], e["repo"], e["pin"]) for e in proposal["provenance"]["evidence"]} + assert ("alert", "example.com/reserved/lmdeploy", FIXED) in cited, "the fixed-pin alert inside the fix is evidence" + assert (base / "loop-propose" / "memory_proposals.jsonl").read_text().count("\n") == 1 + + gate = json.loads((base / "loop-improve" / "gate.json").read_text()) + assert gate["memory_edits"] == [f"rule:{EVAL}:enabled->shadow"] and gate["accepted"] is False and gate["reason"] == "reverted" + assert gate["outcome"]["recall_after"] < gate["outcome"]["recall_before"], "demoting the eval rule loses the benchmark's finding" + assert not (base / "loop-improve" / "rule_policy.json").exists(), "a ledger only when the gate accepts" + assert json.loads((base / "loop-improve" / "journal.json").read_text())[0]["outcome"] == "reverted" + assert not (project / "target" / ".openultrasast").exists(), "the repository's own ledger is never touched" + + report = reconciler.attribution("validation-46") + for row in report["tasks"]: + if row["task"] in (f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS): + assert row["model"] is None and row["usd"] in (0, None) and row["calls"] in (0, None), row + ingested = FileStore(store.root) + outcomes = ingested.rows(kind="proposal_outcome") + assert {r.row["outcome"] for r in outcomes} == {"reverted"} and len(outcomes) == 2, "one per (repo, pin) of the evidence" + [fixed_alert] = ingested.rows(repo="example.com/reserved/lmdeploy", pin=FIXED, kind="alert") + assert fixed_alert.row["in_fix_range"] is True and fixed_alert.row["function"] == "load" diff --git a/tests/test_plane_reconciler.py b/tests/test_plane_reconciler.py index 83497ea..32fa6b5 100644 --- a/tests/test_plane_reconciler.py +++ b/tests/test_plane_reconciler.py @@ -91,6 +91,26 @@ def gone(): sys.exit(1) +def run_task(rec, task, env, spec): + """``python -m openultrasast.plane.tasks.`` with the rendered env, ``AX_TASK_YAML``/``AX_WORKSPACES_YAML``, + each fetched input at its ``OUSAST_INPUT_``, and ``spec["env"]`` (local checkouts for bound Workspaces).""" + import os, subprocess + doc = next(d for d in rec["docs"] if d["kind"] == "Task") + run_env = {{**os.environ, **env, "AX_TASK_YAML": yaml.safe_dump(doc)}} + run_env["AX_WORKSPACES_YAML"] = yaml.safe_dump_all([d for d in rec["docs"] if d["kind"] == "Workspace"]) + for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items(): + run_env["OUSAST_INPUT_" + name] = str(HERE / "fetched" / task / ref) + out = HERE / "out" / task + run_env.update(spec.get("env", {{}}), OUSAST_OUTPUT_DIR=str(out)) + command = doc["spec"]["command"] + argv = [sys.executable, "-m", "openultrasast.plane.tasks." + command[0].replace("-", "_"), *command[1:]] + proc = subprocess.run(argv, env=run_env, capture_output=True, text=True) + (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} exec {{task}} {{proc.returncode}}\\n") + (HERE / "stderr").mkdir(exist_ok=True) + (HERE / "stderr" / task).write_text(proc.stderr) + return {{p.relative_to(out).as_posix(): p.read_text() for p in sorted(out.rglob("*")) if p.is_file()}} + + if argv[:2] == ["apply", "-f"]: docs = list(yaml.safe_load_all(Path(argv[2]).read_text())) for doc in docs: @@ -139,6 +159,8 @@ elif argv[:2] == ["get", "task"]: except urllib.error.HTTPError as exc: # the runner delivers a failed summary naming the input (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} unfetched {{task}} {{name}} {{exc.code}}\\n") files = {{"summary.json": {{"status": "failed", "reason": f"input {{name}} ({{ref}}) not fetched: HTTP {{exc.code}}"}}}} + if "exec" in script and "summary.json" not in files: # the real task module, as the runner runs it + files = run_task(rec, task, env, script["exec"]) buf = io.BytesIO() with tarfile.open(fileobj=buf, mode="w") as tar: for path, content in files.items():