diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 26ea3d7..feb8686 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,7 +1,7 @@ { - "modules": 134, + "modules": 135, "counts": { - "load_bearing": 127, + "load_bearing": 128, "standalone_capability": 7 }, "orphaned": [], @@ -662,6 +662,7 @@ "cli", "improve.memory", "plane.egress", + "plane.engine_alerts", "plane.generate", "plane.memory", "plane.reconciler", @@ -698,6 +699,12 @@ "plane.reconciler" ] }, + { + "module": "plane.engine_alerts", + "classification": "load_bearing", + "reason": "harnessx-removal Req 6.3: engine alerts where quick mode has no rules (PHP), on the host", + "importers": [] + }, { "module": "plane.generate", "classification": "load_bearing", @@ -761,6 +768,7 @@ "reason": "ai-service-plane Req 4.1: the task entrypoint package the runner imports by name; runner integration pending (task 3)", "importers": [ "cli", + "plane.engine_alerts", "plane.generate", "plane.memory", "plane.tasks.agree", @@ -778,7 +786,9 @@ "module": "plane.tasks.alerts", "classification": "load_bearing", "reason": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free", - "importers": [] + "importers": [ + "plane.engine_alerts" + ] }, { "module": "plane.tasks.loop", @@ -1139,6 +1149,7 @@ "improve.memory", "improve.validator", "pairs", + "plane.tasks.alerts", "plane.tasks.loop", "stage_processors" ] diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts-summary.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts-summary.json new file mode 100644 index 0000000..60926f0 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts-summary.json @@ -0,0 +1,96 @@ +{ + "alerts": { + "fixed": 6, + "vulnerable": 6 + }, + "alerts_engine": { + "fixed": 6, + "vulnerable": 6 + }, + "calls": 0, + "coverage": { + "javascript": { + "coverage": "quick", + "files": { + "fixed": 18, + "vulnerable": 18 + } + }, + "php": { + "coverage": "engine", + "files": { + "fixed": 348, + "vulnerable": 348 + } + } + }, + "deadline": 1800.0, + "engine": { + "fixed": { + "bytes": 4256572, + "completed": 130, + "degradations": [ + "cross_partition_semantics_unresolved" + ], + "exit": 0, + "files": 366, + "findings": 6, + "questions": 130, + "seconds": 383.2 + }, + "vulnerable": { + "bytes": 4256094, + "completed": 130, + "degradations": [ + "cross_partition_semantics_unresolved" + ], + "exit": 0, + "files": 366, + "findings": 6, + "questions": 130, + "seconds": 398.2 + } + }, + "engine_languages": [ + "php" + ], + "family": "injection", + "host": true, + "image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "in_fix_range": { + "fixed": 0, + "vulnerable": 0 + }, + "model": null, + "pins": { + "fixed": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", + "vulnerable": "9bf03de643a2edee4642bfc6c94db1f05253da02" + }, + "quick_languages": [ + "c", + "cpp", + "groovy", + "java", + "javascript", + "python", + "typescript" + ], + "read": { + "fixed": { + "bytes": 4256572, + "files": 366 + }, + "vulnerable": { + "bytes": 4256094, + "files": 366 + } + }, + "seconds": 789.1, + "source": "engine", + "status": "done", + "uncovered": [], + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts.jsonl b/benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts.jsonl new file mode 100644 index 0000000..8a6dfad --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts.jsonl @@ -0,0 +1,12 @@ +{"function": "", "in_fix_range": false, "line": 109, "path": "admin_configfiles.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "lockJob", "in_fix_range": false, "line": 232, "path": "lib/Froxlor/Cli/MasterCron.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "safe_exec", "in_fix_range": false, "line": 258, "path": "lib/Froxlor/FileDir.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "safe_exec", "in_fix_range": false, "line": 260, "path": "lib/Froxlor/FileDir.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "checkDatabase", "in_fix_range": false, "line": 364, "path": "lib/Froxlor/Install/Install.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "checkDatabase", "in_fix_range": false, "line": 367, "path": "lib/Froxlor/Install/Install.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "", "in_fix_range": false, "line": 109, "path": "admin_configfiles.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "lockJob", "in_fix_range": false, "line": 232, "path": "lib/Froxlor/Cli/MasterCron.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "safe_exec", "in_fix_range": false, "line": 258, "path": "lib/Froxlor/FileDir.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "safe_exec", "in_fix_range": false, "line": 260, "path": "lib/Froxlor/FileDir.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "checkDatabase", "in_fix_range": false, "line": 364, "path": "lib/Froxlor/Install/Install.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} +{"function": "checkDatabase", "in_fix_range": false, "line": 367, "path": "lib/Froxlor/Install/Install.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/gate.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/gate.json new file mode 100644 index 0000000..266a1cf --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/gate.json @@ -0,0 +1,29 @@ +{ + "accepted": false, + "adoption": null, + "benchmark_edits": [], + "manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml", + "memory_edits": [], + "outcome": { + "accepted": false, + "degradations": [], + "edits": [], + "fp_after": 0.0, + "fp_before": 0.0, + "matched_after": 2, + "matched_before": 2, + "per_profile_after": {}, + "per_profile_before": {}, + "profile_regressions": [], + "profiles_under_minimum": [], + "reason": "no_proposals", + "recall_after": 1.0, + "recall_before": 1.0, + "round": 1, + "score_after": 73, + "score_before": 73 + }, + "proposals": 0, + "reason": "no_proposals", + "round": 1 +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/measure.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/measure.json new file mode 100644 index 0000000..e567bc6 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/measure.json @@ -0,0 +1,170 @@ +{ + "inputs_bytes": { + "froxlor_admin_ip_second_order_sqli": 15137, + "wp_gopay_log_filter_sqli": 4027 + }, + "rows": { + "run": 32, + "snapshot": 191, + "union": 211 + }, + "run": "validation-46", + "run_metrics": { + "agreed": 3, + "alerts": { + "engine:injection": { + "fixed": 6, + "fixed_in_fix_range": 0, + "vulnerable": 6 + } + }, + "candidates": 4, + "cases": 2, + "coverage": { + "javascript": { + "engine": 0, + "none": 0, + "quick": 4 + }, + "php": { + "engine": 4, + "none": 0, + "quick": 0 + } + }, + "declared_sites_agreed": 3, + "declared_sites_matched": 4, + "dispute_rate": { + "injection": { + "candidates": 4, + "disputed": 1, + "rate": 0.25 + } + }, + "disputed": 1, + "rejected": 0, + "tiebreaks": 3, + "usd": 0.10796, + "usd_per_candidate": 0.02699 + }, + "store_metrics": { + "agreed": 28, + "alerts": { + "c-shell-exec": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "c-unsafe-memory-copy": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "python-shell-true": { + "fixed": 1, + "fixed_in_fix_range": 0, + "vulnerable": 1 + }, + "python-unsafe-deserialization": { + "fixed": 8, + "fixed_in_fix_range": 0, + "vulnerable": 8 + }, + "python-unsafe-eval": { + "fixed": 23, + "fixed_in_fix_range": 0, + "vulnerable": 23 + } + }, + "candidates": 43, + "cases": 14, + "coverage": {}, + "declared_sites_agreed": 16, + "declared_sites_matched": 19, + "dispute_rate": { + "injection": { + "candidates": 38, + "disputed": 5, + "rate": 0.1316 + }, + "untrusted_destination": { + "candidates": 5, + "disputed": 0, + "rate": 0.0 + } + }, + "disputed": 5, + "rejected": 10, + "tiebreaks": 10, + "usd": 0.856679, + "usd_per_candidate": 0.019923 + }, + "union_metrics": { + "agreed": 28, + "alerts": { + "c-shell-exec": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "c-unsafe-memory-copy": { + "fixed": 2, + "fixed_in_fix_range": 0, + "vulnerable": 2 + }, + "engine:injection": { + "fixed": 6, + "fixed_in_fix_range": 0, + "vulnerable": 6 + }, + "python-shell-true": { + "fixed": 1, + "fixed_in_fix_range": 0, + "vulnerable": 1 + }, + "python-unsafe-deserialization": { + "fixed": 8, + "fixed_in_fix_range": 0, + "vulnerable": 8 + }, + "python-unsafe-eval": { + "fixed": 23, + "fixed_in_fix_range": 0, + "vulnerable": 23 + } + }, + "candidates": 43, + "cases": 14, + "coverage": { + "javascript": { + "engine": 0, + "none": 0, + "quick": 4 + }, + "php": { + "engine": 4, + "none": 0, + "quick": 0 + } + }, + "declared_sites_agreed": 16, + "declared_sites_matched": 19, + "dispute_rate": { + "injection": { + "candidates": 38, + "disputed": 5, + "rate": 0.1316 + }, + "untrusted_destination": { + "candidates": 5, + "disputed": 0, + "rate": 0.0 + } + }, + "disputed": 5, + "rejected": 10, + "tiebreaks": 10, + "usd": 0.856679, + "usd_per_candidate": 0.019923 + } +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/record.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/record.json new file mode 100644 index 0000000..d532f8c --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/record.json @@ -0,0 +1,391 @@ +{ + "engine_image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "host_resources_during_engine_runs": { + "disk_free_mb_min": 2102, + "engine_container_gib_peak": 2.129, + "host_available_mb_min": 1952, + "host_total_mb": 7877, + "host_used_mb_peak": 5924, + "samples": 61 + }, + "new_spend_usd": 0.0, + "note": "Engine alerts for the two PHP loop cases of validation-46, produced on the host by `ousast plane alerts-engine` (openultrasast:dev, frozen export of f4bd984's src + finding_dump.py, --network none --memory 3g, one container at a time, deadline 1800 s per pin), then the loop's remember -> measure -> propose -> improve on the local ax with the rebuilt runner image. The quick-mode alerts these cases had before (0 per pin) were a zero on an uncovered language.", + "runner_image": "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8", + "tasks": { + "froxlor-admin-ip-second-order-sqli-alerts": { + "finished": "2026-09-30T11:32:02+00:00", + "model": null, + "reason": null, + "reused": { + "alerts": { + "fixed": 6, + "vulnerable": 6 + }, + "host": true, + "image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "seconds": 789.1, + "source": "engine" + }, + "started": "2026-09-30T09:50:49+00:00", + "status": "done", + "summary": { + "alerts": { + "fixed": 6, + "vulnerable": 6 + }, + "alerts_engine": { + "fixed": 6, + "vulnerable": 6 + }, + "calls": 0, + "coverage": { + "javascript": { + "coverage": "quick", + "files": { + "fixed": 18, + "vulnerable": 18 + } + }, + "php": { + "coverage": "engine", + "files": { + "fixed": 348, + "vulnerable": 348 + } + } + }, + "deadline": 1800.0, + "engine": { + "fixed": { + "bytes": 4256572, + "completed": 130, + "degradations": [ + "cross_partition_semantics_unresolved" + ], + "exit": 0, + "files": 366, + "findings": 6, + "questions": 130, + "seconds": 383.2 + }, + "vulnerable": { + "bytes": 4256094, + "completed": 130, + "degradations": [ + "cross_partition_semantics_unresolved" + ], + "exit": 0, + "files": 366, + "findings": 6, + "questions": 130, + "seconds": 398.2 + } + }, + "engine_languages": [ + "php" + ], + "family": "injection", + "host": true, + "image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "in_fix_range": { + "fixed": 0, + "vulnerable": 0 + }, + "model": null, + "pins": { + "fixed": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", + "vulnerable": "9bf03de643a2edee4642bfc6c94db1f05253da02" + }, + "quick_languages": [ + "c", + "cpp", + "groovy", + "java", + "javascript", + "python", + "typescript" + ], + "read": { + "fixed": { + "bytes": 4256572, + "files": 366 + }, + "vulnerable": { + "bytes": 4256094, + "files": 366 + } + }, + "seconds": 789.1, + "source": "engine", + "status": "done", + "uncovered": [], + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 + }, + "usd": null + }, + "froxlor-admin-ip-second-order-sqli-remember": { + "finished": "2026-09-30T11:38:01+00:00", + "model": null, + "reason": null, + "started": "2026-09-30T11:37:32+00:00", + "status": "done", + "summary": { + "calls": 0, + "facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it", + "model": null, + "rows": 25, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + }, + "usd": 0 + }, + "loop-improve": { + "finished": "2026-09-30T11:39:55+00:00", + "model": null, + "reason": null, + "started": "2026-09-30T11:39:24+00:00", + "status": "done", + "summary": { + "accepted": false, + "calls": 0, + "edits": 0, + "memory_edits": 0, + "model": null, + "reason": "no_proposals", + "status": "done", + "step": "improve", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + }, + "usd": 0 + }, + "loop-measure": { + "finished": "2026-09-30T11:38:54+00:00", + "model": null, + "reason": null, + "started": "2026-09-30T11:38:28+00:00", + "status": "done", + "summary": { + "calls": 0, + "model": null, + "rows": { + "run": 32, + "snapshot": 191, + "union": 211 + }, + "status": "done", + "step": "measure", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + }, + "usd": 0 + }, + "loop-propose": { + "finished": "2026-09-30T11:39:24+00:00", + "model": null, + "reason": null, + "started": "2026-09-30T11:38:54+00:00", + "status": "done", + "summary": { + "calls": 0, + "excluded": 0, + "kept": 211, + "model": null, + "proposals": 0, + "rows": 211, + "status": "done", + "step": "propose", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + }, + "usd": 0 + }, + "memory-snapshot": { + "finished": "2026-09-30T11:37:31+00:00", + "model": null, + "reused": { + "index_digest": "5e9299a551eea21289224ee063b2eb8cdf3f845a69967baf8b8c02d5be9cfc74", + "kept": 191, + "rows": 191, + "snapshot": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory" + }, + "started": null, + "status": "done", + "summary": { + "calls": 0, + "model": null, + "reused": { + "excluded": { + "gated_case": {}, + "holdout_pair": {}, + "population": {}, + "rows": 0 + }, + "guard": { + "catalog": "benchmarks/pairs/catalog.toml", + "manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml", + "populations": [] + }, + "index_digest": "5e9299a551eea21289224ee063b2eb8cdf3f845a69967baf8b8c02d5be9cfc74", + "index_entries": 19, + "kept": 191, + "rows": 191, + "store": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory" + }, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": null + }, + "usd": null + }, + "wp-gopay-log-filter-sqli-alerts": { + "finished": "2026-09-30T11:36:07+00:00", + "model": null, + "reason": null, + "reused": { + "alerts": { + "fixed": 0, + "vulnerable": 0 + }, + "host": true, + "image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "seconds": 245.2, + "source": "engine" + }, + "started": "2026-09-30T09:52:27+00:00", + "status": "done", + "summary": { + "alerts": { + "fixed": 0, + "vulnerable": 0 + }, + "alerts_engine": { + "fixed": 0, + "vulnerable": 0 + }, + "calls": 0, + "coverage": { + "javascript": { + "coverage": "quick", + "files": { + "fixed": 4, + "vulnerable": 4 + } + }, + "php": { + "coverage": "engine", + "files": { + "fixed": 15, + "vulnerable": 15 + } + } + }, + "deadline": 1800.0, + "engine": { + "fixed": { + "bytes": 150539, + "completed": 101, + "degradations": [ + "cross_partition_semantics_unresolved", + "vendor_semantics_unresolved" + ], + "exit": 0, + "files": 19, + "findings": 0, + "questions": 101, + "seconds": 122.0 + }, + "vulnerable": { + "bytes": 150528, + "completed": 101, + "degradations": [ + "cross_partition_semantics_unresolved", + "vendor_semantics_unresolved" + ], + "exit": 0, + "files": 19, + "findings": 0, + "questions": 101, + "seconds": 120.2 + } + }, + "engine_languages": [ + "php" + ], + "family": "injection", + "host": true, + "image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "in_fix_range": { + "fixed": 0, + "vulnerable": 0 + }, + "model": null, + "pins": { + "fixed": "7190c985370f9758f226c2f2addd79532a2f93d5", + "vulnerable": "626bc8b5f8d33de6eefd921e3a05ed52ac320bcd" + }, + "quick_languages": [ + "c", + "cpp", + "groovy", + "java", + "javascript", + "python", + "typescript" + ], + "read": { + "fixed": { + "bytes": 150539, + "files": 19 + }, + "vulnerable": { + "bytes": 150528, + "files": 19 + } + }, + "seconds": 245.2, + "source": "engine", + "status": "done", + "uncovered": [], + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 + }, + "usd": null + }, + "wp-gopay-log-filter-sqli-remember": { + "finished": "2026-09-30T11:38:28+00:00", + "model": null, + "reason": null, + "started": "2026-09-30T11:38:01+00:00", + "status": "done", + "summary": { + "calls": 0, + "facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it", + "model": null, + "rows": 7, + "status": "done", + "units_done": 1, + "units_total": 1, + "usage": {}, + "usd": 0 + }, + "usd": 0 + } + } +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/run.yaml b/benchmarks/measurements/2026-09-30-php-engine-alerts/run.yaml new file mode 100644 index 0000000..8d8417f --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/run.yaml @@ -0,0 +1,235 @@ +# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set /tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/phpgap/set/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --loop --loop-commit 1ad472c68d759e4ef010bd7a2e23532b1de538a9 --loop-repo https://github.com/norandom/OpenUltraSAST.git --loop-manifest benchmarks/manifests/java-spring-boot-vulnerable.toml`; do not edit, regenerate. +# 2 cases, 4 candidates (4 kept by the recorded triage); usd ceiling 0.27. +apiVersion: openultrasast.io/v1alpha1 +kind: Run +metadata: + name: validation-46 + annotations: + openultrasast.io/population: population-v2 + openultrasast.io/split: validation +spec: + tasks: + - name: froxlor-admin-ip-second-order-sqli-facts + task: repo-facts-froxlor-admin-ip-second-order-sqli + outputs: + - facts.json + budget: &id001 + usd: 0 + calls: 0 + serialize: repo-facts + - name: froxlor-admin-ip-second-order-sqli-va + task: verify-a-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-vb + task: verify-b-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-agree + task: agree-froxlor-admin-ip-second-order-sqli + inputs: + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: *id001 + - name: wp-gopay-log-filter-sqli-facts + task: repo-facts-wp-gopay-log-filter-sqli + outputs: + - facts.json + budget: &id002 + usd: 0 + calls: 0 + serialize: repo-facts + - name: wp-gopay-log-filter-sqli-va + task: verify-a-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-vb + task: verify-b-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-agree + task: agree-wp-gopay-log-filter-sqli + inputs: + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: *id002 + - name: froxlor-admin-ip-second-order-sqli-vc + task: verify-c-froxlor-admin-ip-second-order-sqli + dependsOn: + - froxlor-admin-ip-second-order-sqli-agree + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + only: froxlor-admin-ip-second-order-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.07 + calls: 16 + - name: froxlor-admin-ip-second-order-sqli-final + task: agree-froxlor-admin-ip-second-order-sqli + inputs: + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: wp-gopay-log-filter-sqli-vc + task: verify-c-wp-gopay-log-filter-sqli + dependsOn: + - wp-gopay-log-filter-sqli-agree + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + only: wp-gopay-log-filter-sqli-agree/disputed.json + outputs: + - units.jsonl + - summary.json + budget: + usd: 0.02 + calls: 8 + - name: wp-gopay-log-filter-sqli-final + task: agree-wp-gopay-log-filter-sqli + inputs: + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl + outputs: + - agreed.json + - disputed.json + budget: + usd: 0 + calls: 0 + - name: froxlor-admin-ip-second-order-sqli-alerts + task: alerts-froxlor-admin-ip-second-order-sqli + outputs: + - alerts.jsonl + - summary.json + budget: + usd: 0 + calls: 0 + serialize: engine + - name: froxlor-admin-ip-second-order-sqli-remember + task: remember-froxlor-admin-ip-second-order-sqli + inputs: + facts: froxlor-admin-ip-second-order-sqli-facts/facts.json + facts_summary: froxlor-admin-ip-second-order-sqli-facts/summary.json + pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl + summary_a: froxlor-admin-ip-second-order-sqli-va/summary.json + pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl + summary_b: froxlor-admin-ip-second-order-sqli-vb/summary.json + pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl + summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json + agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json + alerts: froxlor-admin-ip-second-order-sqli-alerts/alerts.jsonl + alerts_summary: froxlor-admin-ip-second-order-sqli-alerts/summary.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: wp-gopay-log-filter-sqli-alerts + task: alerts-wp-gopay-log-filter-sqli + outputs: + - alerts.jsonl + - summary.json + budget: + usd: 0 + calls: 0 + serialize: engine + - name: wp-gopay-log-filter-sqli-remember + task: remember-wp-gopay-log-filter-sqli + inputs: + facts: wp-gopay-log-filter-sqli-facts/facts.json + facts_summary: wp-gopay-log-filter-sqli-facts/summary.json + pass_a: wp-gopay-log-filter-sqli-va/units.jsonl + summary_a: wp-gopay-log-filter-sqli-va/summary.json + pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl + summary_b: wp-gopay-log-filter-sqli-vb/summary.json + pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl + summary_c: wp-gopay-log-filter-sqli-vc/summary.json + agreed: wp-gopay-log-filter-sqli-final/agreed.json + alerts: wp-gopay-log-filter-sqli-alerts/alerts.jsonl + alerts_summary: wp-gopay-log-filter-sqli-alerts/summary.json + outputs: + - memory.jsonl + budget: + usd: 0 + calls: 0 + - name: memory-snapshot + task: memory-snapshot + outputs: + - rows.jsonl + - index.json + budget: + usd: 0 + calls: 0 + - name: loop-measure + task: loop-measure + inputs: + remember_froxlor-admin-ip-second-order-sqli: froxlor-admin-ip-second-order-sqli-remember/memory.jsonl + remember_wp-gopay-log-filter-sqli: wp-gopay-log-filter-sqli-remember/memory.jsonl + snapshot: memory-snapshot/rows.jsonl + outputs: + - measure.json + - rows.jsonl + budget: + usd: 0 + calls: 0 + - name: loop-propose + task: loop-propose + inputs: + measure: loop-measure/measure.json + rows: loop-measure/rows.jsonl + index: memory-snapshot/index.json + outputs: + - proposals.jsonl + - memory_proposals.jsonl + - signals.json + budget: + usd: 0 + calls: 0 + - name: loop-improve + task: loop-improve + inputs: + proposals: loop-propose/proposals.jsonl + outputs: + - gate.json + - journal.json + budget: + usd: 0 + calls: 0 + serialize: engine diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/set.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/set.json new file mode 100644 index 0000000..5e402b3 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/set.json @@ -0,0 +1,22 @@ +{ + "wp-gopay-log-filter-sqli": [ + [ + "admin/views/log.php", + "" + ] + ], + "froxlor-admin-ip-second-order-sqli": [ + [ + "lib/Froxlor/Api/Commands/Domains.php", + "validateIpAddresses" + ], + [ + "lib/Froxlor/Api/Commands/IpsAndPorts.php", + "listing" + ], + [ + "lib/Froxlor/Api/Commands/IpsAndPorts.php", + "listingCount" + ] + ] +} \ No newline at end of file diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/signals.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/signals.json new file mode 100644 index 0000000..90ee97f --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/signals.json @@ -0,0 +1,8 @@ +{ + "advisory": true, + "disputed_families": [], + "thresholds": { + "min_candidates": 5, + "min_dispute_rate": 0.3 + } +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/snapshot-index.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/snapshot-index.json new file mode 100644 index 0000000..e90d067 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/snapshot-index.json @@ -0,0 +1,18 @@ +{ + "excluded": { + "gated_case": {}, + "holdout_pair": {}, + "population": {}, + "rows": 0 + }, + "guard": { + "catalog": "benchmarks/pairs/catalog.toml", + "manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml", + "populations": [] + }, + "index_digest": "5e9299a551eea21289224ee063b2eb8cdf3f845a69967baf8b8c02d5be9cfc74", + "index_entries": 19, + "kept": 191, + "rows": 191, + "store": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory" +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts-summary.json b/benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts-summary.json new file mode 100644 index 0000000..6929ba1 --- /dev/null +++ b/benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts-summary.json @@ -0,0 +1,98 @@ +{ + "alerts": { + "fixed": 0, + "vulnerable": 0 + }, + "alerts_engine": { + "fixed": 0, + "vulnerable": 0 + }, + "calls": 0, + "coverage": { + "javascript": { + "coverage": "quick", + "files": { + "fixed": 4, + "vulnerable": 4 + } + }, + "php": { + "coverage": "engine", + "files": { + "fixed": 15, + "vulnerable": 15 + } + } + }, + "deadline": 1800.0, + "engine": { + "fixed": { + "bytes": 150539, + "completed": 101, + "degradations": [ + "cross_partition_semantics_unresolved", + "vendor_semantics_unresolved" + ], + "exit": 0, + "files": 19, + "findings": 0, + "questions": 101, + "seconds": 122.0 + }, + "vulnerable": { + "bytes": 150528, + "completed": 101, + "degradations": [ + "cross_partition_semantics_unresolved", + "vendor_semantics_unresolved" + ], + "exit": 0, + "files": 19, + "findings": 0, + "questions": 101, + "seconds": 120.2 + } + }, + "engine_languages": [ + "php" + ], + "family": "injection", + "host": true, + "image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb", + "in_fix_range": { + "fixed": 0, + "vulnerable": 0 + }, + "model": null, + "pins": { + "fixed": "7190c985370f9758f226c2f2addd79532a2f93d5", + "vulnerable": "626bc8b5f8d33de6eefd921e3a05ed52ac320bcd" + }, + "quick_languages": [ + "c", + "cpp", + "groovy", + "java", + "javascript", + "python", + "typescript" + ], + "read": { + "fixed": { + "bytes": 150539, + "files": 19 + }, + "vulnerable": { + "bytes": 150528, + "files": 19 + } + }, + "seconds": 245.2, + "source": "engine", + "status": "done", + "uncovered": [], + "units_done": 2, + "units_total": 2, + "usage": {}, + "usd": 0 +} diff --git a/benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts.jsonl b/benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/plane/tasks/agree.yaml b/plane/tasks/agree.yaml index f3920bf..24a99ce 100644 --- a/plane/tasks/agree.yaml +++ b/plane/tasks/agree.yaml @@ -7,7 +7,7 @@ metadata: name: agree atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8" command: ["agree"] resources: requests: diff --git a/plane/tasks/alerts.yaml b/plane/tasks/alerts.yaml index d17bd39..984ec03 100644 --- a/plane/tasks/alerts.yaml +++ b/plane/tasks/alerts.yaml @@ -7,7 +7,7 @@ metadata: name: alerts atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8" command: ["alerts"] resources: requests: diff --git a/plane/tasks/loop.yaml b/plane/tasks/loop.yaml index d936d9e..7e3afe1 100644 --- a/plane/tasks/loop.yaml +++ b/plane/tasks/loop.yaml @@ -7,7 +7,7 @@ metadata: name: loop atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8" command: ["loop"] resources: requests: diff --git a/plane/tasks/remember.yaml b/plane/tasks/remember.yaml index d11e9cb..1327e01 100644 --- a/plane/tasks/remember.yaml +++ b/plane/tasks/remember.yaml @@ -7,7 +7,7 @@ metadata: name: remember atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8" command: ["remember"] resources: requests: diff --git a/plane/tasks/repo-facts.yaml b/plane/tasks/repo-facts.yaml index 3d642f4..38fd2e4 100644 --- a/plane/tasks/repo-facts.yaml +++ b/plane/tasks/repo-facts.yaml @@ -8,7 +8,7 @@ metadata: name: repo-facts atespace: default spec: - image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8" command: ["repo-facts"] resources: requests: diff --git a/plane/tasks/validation-46.yaml b/plane/tasks/validation-46.yaml index 48a0231..2884241 100644 --- a/plane/tasks/validation-46.yaml +++ b/plane/tasks/validation-46.yaml @@ -5,9 +5,9 @@ metadata: name: repo-facts-budibase-mongo-template-nosqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}' + openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -36,7 +36,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -67,7 +67,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -96,7 +96,7 @@ metadata: name: agree-budibase-mongo-template-nosqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -123,9 +123,9 @@ metadata: name: repo-facts-cct-studio-server-cmdi atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}' + openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -154,7 +154,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -185,7 +185,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -214,7 +214,7 @@ metadata: name: agree-cct-studio-server-cmdi atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -241,9 +241,9 @@ metadata: name: repo-facts-contextforge-prompt-ssti atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}' + openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -272,7 +272,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -303,7 +303,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -332,7 +332,7 @@ metadata: name: agree-contextforge-prompt-ssti atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -359,9 +359,9 @@ metadata: name: repo-facts-froxlor-admin-ip-second-order-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}' + openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -390,7 +390,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -421,7 +421,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -450,7 +450,7 @@ metadata: name: agree-froxlor-admin-ip-second-order-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -477,9 +477,9 @@ metadata: name: repo-facts-fuxa-tdengine-escape-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}' + openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -508,7 +508,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -539,7 +539,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -568,7 +568,7 @@ metadata: name: agree-fuxa-tdengine-escape-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -595,9 +595,9 @@ metadata: name: repo-facts-lmdeploy-media-url-ssrf atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}' + openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -626,7 +626,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -657,7 +657,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -686,7 +686,7 @@ metadata: name: agree-lmdeploy-media-url-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -713,9 +713,9 @@ metadata: name: repo-facts-medplum-external-auth-open-redirect atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}' + openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -744,7 +744,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -775,7 +775,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -804,7 +804,7 @@ metadata: name: agree-medplum-external-auth-open-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -831,9 +831,9 @@ metadata: name: repo-facts-pgadmin-maintenance-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}' + openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -862,7 +862,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -893,7 +893,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -922,7 +922,7 @@ metadata: name: agree-pgadmin-maintenance-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -949,9 +949,9 @@ metadata: name: repo-facts-qwed-sympy-eval atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}' + openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -980,7 +980,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1011,7 +1011,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1040,7 +1040,7 @@ metadata: name: agree-qwed-sympy-eval atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1067,9 +1067,9 @@ metadata: name: repo-facts-reactpress-findall-column-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}' + openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -1098,7 +1098,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1129,7 +1129,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1158,7 +1158,7 @@ metadata: name: agree-reactpress-findall-column-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1185,9 +1185,9 @@ metadata: name: repo-facts-unleash-addon-webhook-ssrf atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}' + openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -1216,7 +1216,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1247,7 +1247,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1276,7 +1276,7 @@ metadata: name: agree-unleash-addon-webhook-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1303,9 +1303,9 @@ metadata: name: repo-facts-winml-cli-serve-cors atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}' + openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -1334,7 +1334,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1365,7 +1365,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1394,7 +1394,7 @@ metadata: name: agree-winml-cli-serve-cors atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1421,9 +1421,9 @@ metadata: name: repo-facts-wp-directorist-avatar-ssrf atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}' + openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -1452,7 +1452,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1483,7 +1483,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1512,7 +1512,7 @@ metadata: name: agree-wp-directorist-avatar-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1539,9 +1539,9 @@ metadata: name: repo-facts-wp-gopay-log-filter-sqli atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}' + openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -1570,7 +1570,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1601,7 +1601,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1630,7 +1630,7 @@ metadata: name: agree-wp-gopay-log-filter-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1657,9 +1657,9 @@ metadata: name: repo-facts-wp-groundhogg-confirm-redirect atespace: default annotations: - openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}' + openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}' spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - repo-facts env: @@ -1688,7 +1688,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1719,7 +1719,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1748,7 +1748,7 @@ metadata: name: agree-wp-groundhogg-confirm-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - agree env: @@ -1777,7 +1777,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1808,7 +1808,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1839,7 +1839,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1870,7 +1870,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1901,7 +1901,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1932,7 +1932,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1963,7 +1963,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -1994,7 +1994,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2025,7 +2025,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2056,7 +2056,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2087,7 +2087,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2118,7 +2118,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2149,7 +2149,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2180,7 +2180,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2211,7 +2211,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - verify env: @@ -2240,7 +2240,7 @@ metadata: name: remember-budibase-mongo-template-nosqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2265,7 +2265,7 @@ metadata: name: remember-cct-studio-server-cmdi atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2290,7 +2290,7 @@ metadata: name: remember-contextforge-prompt-ssti atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2315,7 +2315,7 @@ metadata: name: remember-froxlor-admin-ip-second-order-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2340,7 +2340,7 @@ metadata: name: remember-fuxa-tdengine-escape-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2365,7 +2365,7 @@ metadata: name: remember-lmdeploy-media-url-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2390,7 +2390,7 @@ metadata: name: remember-medplum-external-auth-open-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2415,7 +2415,7 @@ metadata: name: remember-pgadmin-maintenance-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2440,7 +2440,7 @@ metadata: name: remember-qwed-sympy-eval atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2465,7 +2465,7 @@ metadata: name: remember-reactpress-findall-column-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2490,7 +2490,7 @@ metadata: name: remember-unleash-addon-webhook-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2515,7 +2515,7 @@ metadata: name: remember-winml-cli-serve-cors atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2540,7 +2540,7 @@ metadata: name: remember-wp-directorist-avatar-ssrf atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2565,7 +2565,7 @@ metadata: name: remember-wp-gopay-log-filter-sqli atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: @@ -2590,7 +2590,7 @@ metadata: name: remember-wp-groundhogg-confirm-redirect atespace: default spec: - image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9 + image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8 command: - remember env: diff --git a/plane/tasks/verify.yaml b/plane/tasks/verify.yaml index 00721a7..7dbefe9 100644 --- a/plane/tasks/verify.yaml +++ b/plane/tasks/verify.yaml @@ -10,7 +10,7 @@ metadata: annotations: openultrasast.io/model: deepseek-flash spec: - image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9" + image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8" command: ["verify"] resources: requests: diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index d8dc1a4..7f3e312 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -274,6 +274,18 @@ def _main(argv: list[str] | None) -> int: plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/.yaml names population and split") plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)") plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)") + plane_engine = plane_sub.add_parser( + "alerts-engine", help="a Run's `alerts` for languages quick mode does not cover (PHP), from the engine image on this host" + ) + plane_engine.add_argument("run_manifest", type=Path) + plane_engine.add_argument( + "--source", type=Path, required=True, help="a frozen export: git archive HEAD src benchmarks/push/finding_dump.py" + ) + plane_engine.add_argument("--cases", default="", help="comma-separated case ids (default: every alerts case of the Run)") + plane_engine.add_argument("--deadline", type=float, default=1800.0, help="the engine's scan deadline per pin, seconds") + plane_engine.add_argument("--image", default="openultrasast:dev") + plane_engine.add_argument("--repos", type=Path, help="the case cache (default ~/.cache/openultrasast/independent)") + plane_engine.add_argument("--work", type=Path, help="where pins are exported and results land (default: a temporary directory)") plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file") plane_ws.add_argument("population", type=Path) plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces")) @@ -1318,6 +1330,8 @@ def _plane(args: argparse.Namespace) -> int: if args.plane_command == "status": print(reconciler.status(args.run, units=args.units)) return 0 + if args.plane_command == "alerts-engine": + return _plane_alerts_engine(args) if args.plane_command == "doctor": checks = reconciler.doctor() for name, ok, text in checks: @@ -1332,6 +1346,24 @@ def _plane(args: argparse.Namespace) -> int: return 0 +def _plane_alerts_engine(args: argparse.Namespace) -> int: + from .plane import engine_alerts + + only = [c.strip() for c in args.cases.split(",") if c.strip()] + try: + results = engine_alerts.alerts_engine( + args.run_manifest, source=args.source, only=only, work=args.work, cache=args.repos or engine_alerts.CACHE, + deadline=args.deadline, image=args.image, + ) # fmt: skip + except (engine_alerts.EngineAlertsError, OSError, ValueError, subprocess.CalledProcessError) as exc: + print(f"alerts-engine failed: {exc}", file=sys.stderr) + return 2 + keys = ("case", "status", "reason", "alerts", "alerts_engine", "in_fix_range", "engine", "uncovered", "seconds") + for result in results: + print(json.dumps({k: result[k] for k in keys if k in result}, sort_keys=True)) + return 0 + + def _plane_memory( step: str, run_manifest: Path | None, *, run: str | None = None, population: str | None = None, split: str | None = None ) -> int: diff --git a/src/openultrasast/improve/memory.py b/src/openultrasast/improve/memory.py index d5c0987..8ed5331 100644 --- a/src/openultrasast/improve/memory.py +++ b/src/openultrasast/improve/memory.py @@ -19,6 +19,11 @@ final counts as no evidence in either direction. - **M3, disputed families**: families with at least :data:`M3_MIN_CANDIDATES` candidates and a dispute rate of at least :data:`M3_MIN_DISPUTE_RATE`. Advisory (``signals.json``); never an edit, never shown to the validator. +**Coverage**: M1 and M2 skip every repository and pin whose ``coverage`` rows record the rule's language only as +``none`` (quick mode had no rule for it and no engine alerts were produced): a rule cannot be demoted or promoted +on evidence that could not exist. The skipped alerts are counted per ``repo@pin`` under ``coverage_none`` in the +proposal's ``excluded``. + **Train-on-test guard** (:class:`Guard`), applied before any rule sees a row: a row is dropped when its population is one the proposal will be qualified on, its (repo, pin) is a case of the gated benchmark manifest, or its repository is a holdout pair's repository. The pair catalog records a holdout pair's *fix* commit, not the @@ -142,6 +147,7 @@ class _View: verdicts: dict[tuple[str, str, str], Record] = field(default_factory=dict) # (repo, pin, candidate) sites: set[tuple[str, str]] = field(default_factory=set) # (repo, candidate) with site_match true alerts: dict[str, list[Record]] = field(default_factory=dict) # rule_id -> alerts + coverage: dict[tuple[str, str, str], set[str]] = field(default_factory=dict) # (repo, pin, language) -> recorded kinds @classmethod def of(cls, records: Sequence[Record]) -> _View: @@ -151,7 +157,9 @@ class _View: seen: set[tuple[object, ...]] = set() for rec in records: row = rec.row - if row["kind"] == "verdict": + if row["kind"] == "coverage" and row.get("language"): + view.coverage.setdefault((row["repo"], row["pin"], str(row["language"])), set()).add(str(row.get("coverage"))) + elif row["kind"] == "verdict": view.verdicts[(row["repo"], row["pin"], str(row.get("candidate")))] = rec if row.get("site_match") is True: view.sites.add((row["repo"], str(row.get("candidate")))) @@ -171,6 +179,26 @@ class _View: flag = alert.get("in_fix_range") return bool(flag) if flag is not None else (alert["repo"], _cand(alert)) in self.sites + def blind(self, repo: str, pin: str, languages: Iterable[str]) -> bool: + """True when a language of the rule was recorded at this repository and pin with coverage ``none`` only: no + rule could fire there, so what the rule did or did not say there is no evidence either way.""" + return any(self.coverage.get((repo, pin, language)) == {"none"} for language in languages) + + def seeing(self, languages_by_rule: Mapping[str, Sequence[str]]) -> tuple[_View, dict[str, int]]: + """This view without the alerts at a repository and pin blind to their rule's language, and how many were + dropped per ``repo@pin``.""" + kept: dict[str, list[Record]] = {} + dropped: dict[str, int] = {} + for rule_id, alerts in self.alerts.items(): + languages = languages_by_rule.get(rule_id, ()) + for alert in alerts: + if self.blind(alert.row["repo"], alert.row["pin"], languages): + label = f"{alert.row['repo']}@{alert.row['pin']}" + dropped[label] = dropped.get(label, 0) + 1 + else: + kept.setdefault(rule_id, []).append(alert) + return _View(self.verdicts, self.sites, kept, self.coverage), dict(sorted(dropped.items())) + def _m1(rule_id: str, view: _View) -> tuple[list[Record], dict[str, int]] | None: evidence: list[Record] = [] @@ -262,7 +290,9 @@ def propose_from_memory( ) -> list[MemoryProposal]: """M1 and M2 over the rows the guard keeps, minus proposals already reverted on the same evidence.""" kept, dropped = excluded.apply(rows) - view = _View.of(kept) + view, blind = _View.of(kept).seeing({rid: rule.languages for rid, rule in ruleset_by_id.items()}) + if blind: + dropped = {**dropped, "coverage_none": blind} statuses = {rid: _status(rid, ruleset_by_id, current_ledger) for rid in ruleset_by_id} enabled = {rid for rid, status in statuses.items() if status == "enabled"} proposals: list[MemoryProposal] = [] diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index 6bc6af6..71120bf 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -66,6 +66,7 @@ SPEC_OWNED = { "plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`", "plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free", "plane.tasks.alerts": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free", + "plane.engine_alerts": "harnessx-removal Req 6.3: engine alerts where quick mode has no rules (PHP), on the host", "plane.tasks.loop": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps", "plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`", "plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory", diff --git a/src/openultrasast/plane/engine_alerts.py b/src/openultrasast/plane/engine_alerts.py new file mode 100644 index 0000000..c74801e --- /dev/null +++ b/src/openultrasast/plane/engine_alerts.py @@ -0,0 +1,257 @@ +"""Engine alerts for the languages quick mode does not cover, produced on the host (harnessx-removal Req 6.3). + +The loop's ``alerts`` task runs the quick-mode scan, which has rules only for the languages its ruleset declares; a +PHP case yields zero alerts that mean nothing (``coverage: none``). PHP detection comes from the Joern taint engine +(``ruleset/semantic/php.toml``), which runs in the ``openultrasast:dev`` image, not in the runner sandbox. So +``ousast plane alerts-engine `` does the ``alerts`` task's work on the host for every case whose files +include a language quick mode does not cover but the engine does: + +- export the case's vulnerable and fixed pins from the case cache (``git archive``, as + ``benchmarks/independent/evaluate.py`` does), run the quick scan on each (:func:`.tasks.alerts.scan`) and the + engine container on each -- ``benchmarks/push/finding_dump.py`` from a FROZEN source export (never a live tree), + ``--network none --memory 3g``, the case's family, one container at a time, a per-case deadline; +- convert the engine's findings into the ``alerts.jsonl`` row shape of the ``alerts`` task (``rule_id`` is + ``engine:``, ``source: "engine"``, ``in_fix_range`` from ``case.json``'s ranges), write both kinds of rows + and a ``summary.json`` (``coverage`` with the engine's languages as ``engine``, the engine's files read, questions, + completed, seconds and degradations per pin) into ``/-alerts/``; +- mark the task done with :func:`.reconciler.mark_done`, the mechanism fact reuse uses, so the Run skips it. + +The instrument must have read its input: no result file, no ``read N files`` line, or ``questions == 0`` fails the +case loudly (:class:`EngineAlertsError`) and leaves the run directory untouched. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import tempfile +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import reconciler +from .tasks.alerts import ROLES, coverage, covers, engine_languages, function_at, in_range, quick_languages, scan, uncovered +from .tasks.alerts import write_rows as write_alert_rows +from .tasks.alerts import write_summary as write_alert_summary + +IMAGE = "openultrasast:dev" +MEMORY = "3g" +DEADLINE = 1800.0 +GRACE = 600.0 # the container's own startup and teardown beyond the scan deadline +CACHE = Path.home() / ".cache" / "openultrasast" / "independent" +DUMP = Path("benchmarks") / "push" / "finding_dump.py" +_READ = re.compile(r"^read (\d+) files, (\d+) bytes", re.MULTILINE) +_SITE = re.compile(r"^(.+?):(\d+):(.*)$") + +Runner = Callable[[Sequence[str], float], subprocess.CompletedProcess[str]] + + +class EngineAlertsError(RuntimeError): + """The engine did not demonstrably read its input, or a case cannot be prepared.""" + + +def run_command(command: Sequence[str], timeout: float) -> subprocess.CompletedProcess[str]: + return subprocess.run(list(command), capture_output=True, text=True, timeout=timeout, check=False) + + +@dataclass(frozen=True) +class EngineCase: + case_id: str + entry: str # the Run entry of the case's `alerts` task + repo: str + pins: Mapping[str, str] # role -> commit + family: str + ranges: Mapping[str, Any] # role -> {path: [[lo, hi], ...]} or None + + +def engine_cases(run_manifest: Path) -> tuple[str, list[EngineCase]]: + """(Run name, every `alerts` entry of the Run as a case): pins from the task's env, family and fix ranges from the + `case.json` of its bound inputs Workspace, the repository from its bound git Workspace.""" + run, manifests = reconciler.load_run(Path(run_manifest)) + cases: list[EngineCase] = [] + for entry in run.tasks: + task = manifests.tasks[entry.task] + if tuple(task.command[:1]) != ("alerts",): + continue + env = {e.name: e.value for e in task.env} + bound = [manifests.workspaces[b.name] for b in task.workspaces if b.name in manifests.workspaces] + files = {f.path: f.content for ws in bound for f in ws.files} + repos = [g.repo for ws in bound for g in ws.git] + if "case.json" not in files or not repos: + raise EngineAlertsError(f"{entry.name}: no bound case.json or git Workspace") + case = json.loads(files["case.json"]) + pins = {"vulnerable": env.get("OUSAST_VULNERABLE_PIN", ""), "fixed": env.get("OUSAST_FIXED_PIN", "")} + if not all(pins.values()) or not case.get("family"): + raise EngineAlertsError(f"{entry.name}: pins {pins} or family {case.get('family')!r} missing") + case_id = str(case.get("id") or entry.name.removesuffix("-alerts")) + ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")} + cases.append(EngineCase(case_id, entry.name, repos[0], pins, str(case["family"]), ranges)) + return run.metadata.name, cases + + +def frozen_source(source: Path) -> Path: + """The analyzer tree the engine container mounts: an export (``git archive HEAD src benchmarks/push/finding_dump.py``), + never a live checkout, whose files could change under a long run.""" + source = Path(source).resolve() + if (source / ".git").exists(): + raise EngineAlertsError(f"{source} is a git checkout; export it first (git archive HEAD src {DUMP.as_posix()})") + if not (source / "src" / "openultrasast").is_dir() or not (source / DUMP).is_file(): + raise EngineAlertsError(f"{source} holds no src/openultrasast and {DUMP.as_posix()}") + return source + + +def export_pin(cache_repo: Path, pin: str, target: Path) -> Path: + """``git archive `` of the case cache into ``target`` (emptied first).""" + if target.exists(): + shutil.rmtree(target) + target.mkdir(parents=True) + archive = target.parent / f"{target.name}.tar" + with archive.open("wb") as stream: + subprocess.run(["git", "-C", str(cache_repo), "archive", pin], stdout=stream, check=True) + subprocess.run(["tar", "-xf", str(archive), "-C", str(target)], check=True) + archive.unlink() + if not any(target.rglob("*")): + raise EngineAlertsError(f"{cache_repo} @ {pin}: the export is empty") + return target + + +def docker_command(source: Path, checkout: Path, out_dir: Path, name: str, family: str, deadline: float, image: str) -> list[str]: + return [ + "docker", "run", "--rm", "--init", "--network", "none", "--memory", MEMORY, "--entrypoint", "python", + "-e", "OUSAST_LOG_LEVEL=WARNING", "-e", "PYTHONPATH=/new/src", + "-v", f"{source / 'src'}:/new/src:ro", "-v", f"{source / DUMP}:/dump.py:ro", + "-v", f"{checkout}:/case:ro", "-v", f"{out_dir}:/out", + image, "/dump.py", "--root", "/case", "--families", family, + "--out", f"/out/{name}.json", "--deadline", str(int(deadline)), "--max-regions", "5000", + ] # fmt: skip + + +def read_record(out: Path, log: str, returncode: int) -> dict[str, Any]: + """The engine's result, with the files and bytes it said it read; fails unless it read input and asked questions.""" + tail = log.strip()[-600:] + if not out.is_file(): + raise EngineAlertsError(f"the engine wrote no result {out.name} (exit {returncode}): {tail}") + record = json.loads(out.read_text(encoding="utf-8")) + read = _READ.search(log) + if read is None or int(read.group(1)) == 0: + raise EngineAlertsError(f"{out.name}: the engine did not report reading any file (exit {returncode}): {tail}") + if not record.get("questions"): + raise EngineAlertsError(f"{out.name}: no question was asked (exit {returncode}): the engine measured nothing") + return {**record, "files": int(read.group(1)), "bytes": int(read.group(2)), "exit": returncode} + + +def parse_site(site: str) -> tuple[str, int | None, str | None]: + match = _SITE.match(site) + if match is None: + return site, None, None + return match.group(1), int(match.group(2)), match.group(3) or None + + +def engine_rows(record: Mapping[str, Any], root: Path, role: str, pin: str, ranges: Mapping[str, Any] | None) -> list[dict[str, Any]]: + """The engine's findings as ``alerts.jsonl`` rows (the ``alerts`` task's shape plus ``source: "engine"``), one per + rule, file, line; ``function`` by the ``repo-facts`` patterns when ``root`` holds the file, else the engine's.""" + cache: dict[str, list[str]] = {} + rows: dict[tuple[str, str, int | None], dict[str, Any]] = {} + for finding in record.get("findings") or []: + path, line, function = parse_site(str(finding["site"])) + rule_id = f"engine:{finding['family']}" + row = { + "rule_id": rule_id, "rule_status": "enabled", "path": path, "line": line, + "function": function_at(root, path, line, function, cache), "pin_role": role, "pin": pin, + "in_fix_range": in_range(ranges, path, line), "source": "engine", + } # fmt: skip + rows.setdefault((rule_id, path, line), row) + return sorted(rows.values(), key=lambda r: (r["path"], r["line"] or 0, r["rule_id"])) + + +def _image_id(image: str, runner: Runner) -> str: + done = runner(["docker", "image", "inspect", "--format", "{{.Id}}", image], 60.0) + return done.stdout.strip() if done.returncode == 0 and done.stdout.strip() else image + + +def produce( + case: EngineCase, run_name: str, source: Path, work: Path, *, cache: Path = CACHE, deadline: float = DEADLINE, + image: str = IMAGE, runner: Runner | None = None, results_root: Path | None = None, +) -> dict[str, Any]: # fmt: skip + """Quick and engine alerts of one case on both pins into the run directory, the task marked done; the summary. + A case whose files are all in quick-covered languages is ``skipped`` and nothing is written.""" + runner = runner or run_command + started = time.monotonic() + quick, engine = quick_languages(), engine_languages() + rows: list[dict[str, Any]] = [] + files: dict[str, dict[str, int]] = {} + summary: dict[str, Any] = {"read": {}, "alerts": {}, "alerts_engine": {}, "in_fix_range": {}, "engine": {}, "pins": dict(case.pins)} + needed: set[str] = set() + for role in ROLES: + checkout = export_pin(cache / case.case_id, case.pins[role], work / f"{case.case_id}--{role}") + try: + quick_rows, read = scan(checkout, role, case.pins[role], case.ranges.get(role)) + files[role] = read.pop("languages") + if read["files"] == 0: + raise EngineAlertsError(f"{case.case_id} {role}: the quick scan read no file of {checkout}") + needed |= {lang for lang in files[role] if lang != "unknown" and not covers(quick, lang) and lang in engine} + if not needed: + return {"case": case.case_id, "status": "skipped", "reason": "quick mode covers every language read", "languages": files} + out_dir = work / "out" + out_dir.mkdir(parents=True, exist_ok=True) + name = f"{case.case_id}--{role}" + (out_dir / f"{name}.json").unlink(missing_ok=True) + command = docker_command(source, checkout, out_dir, name, case.family, deadline, image) + try: + done = runner(command, deadline + GRACE) + except subprocess.TimeoutExpired as exc: + raise EngineAlertsError(f"{name}: the engine container outlived {deadline + GRACE:g}s") from exc + record = read_record(out_dir / f"{name}.json", (done.stdout or "") + "\n" + (done.stderr or ""), done.returncode) + found = engine_rows(record, checkout, role, case.pins[role], case.ranges.get(role)) + finally: + shutil.rmtree(checkout, ignore_errors=True) + rows.extend(quick_rows + found) + summary["read"][role] = read + summary["alerts"][role] = len(quick_rows) + len(found) + summary["alerts_engine"][role] = len(found) + summary["in_fix_range"][role] = sum(1 for r in found if r["in_fix_range"] is True) + keys = ("files", "bytes", "questions", "completed", "seconds", "degradations", "exit") + summary["engine"][role] = {**{k: record.get(k) for k in keys}, "findings": len(record.get("findings") or [])} + cov = coverage(files, quick, needed) + summary.update( + status="done", units_done=len(ROLES), units_total=len(ROLES), usd=0, calls=0, usage={}, model=None, source="engine", + host=True, family=case.family, image=_image_id(image, runner), deadline=deadline, quick_languages=sorted(quick), + engine_languages=sorted(needed), coverage=cov, uncovered=uncovered(cov), seconds=round(time.monotonic() - started, 1), + ) # fmt: skip + output = reconciler.run_dir(run_name, results_root) / case.entry + output.mkdir(parents=True, exist_ok=True) + write_alert_rows(output, rows) + write_alert_summary(output, summary) + reused = {"source": "engine", "host": True, "image": summary["image"], "alerts": summary["alerts"], "seconds": summary["seconds"]} + reconciler.mark_done(run_name, case.entry, reused=reused, results_root=results_root) + return {"case": case.case_id, **summary} + + +def alerts_engine( + run_manifest: Path, *, source: Path, only: Sequence[str] = (), work: Path | None = None, cache: Path = CACHE, + deadline: float = DEADLINE, image: str = IMAGE, runner: Runner | None = None, results_root: Path | None = None, +) -> list[dict[str, Any]]: # fmt: skip + """:func:`produce` for every `alerts` case of the Run (or those named), one engine container at a time.""" + run_name, cases = engine_cases(run_manifest) + unknown = set(only) - {c.case_id for c in cases} + if unknown: + raise EngineAlertsError(f"no alerts task in {run_manifest} for {sorted(unknown)}") + frozen = frozen_source(source) + scratch = Path(work) if work is not None else Path(tempfile.mkdtemp(prefix="ousast-engine-alerts-")) + results: list[dict[str, Any]] = [] + for case in cases: + if only and case.case_id not in only: + continue + results.append( + produce(case, run_name, frozen, scratch, cache=cache, deadline=deadline, image=image, runner=runner, results_root=results_root) + ) + if work is None: + shutil.rmtree(scratch, ignore_errors=True) + return results + + +__all__ = ["EngineAlertsError", "EngineCase", "alerts_engine", "engine_cases", "engine_rows", "parse_site", "produce", "read_record"] diff --git a/src/openultrasast/plane/generate.py b/src/openultrasast/plane/generate.py index 6f3bec3..3cffc7f 100644 --- a/src/openultrasast/plane/generate.py +++ b/src/openultrasast/plane/generate.py @@ -314,6 +314,7 @@ def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, inputs["agreed"] = f"{case_id}-final/agreed.json" if alerts: inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl" + inputs["alerts_summary"] = f"{case_id}-alerts/summary.json" # the coverage: whether a zero could mean clean entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()} return task, entry @@ -332,7 +333,7 @@ def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict "OUSAST_FIXED_PIN": str(case["fixed"]), } # fmt: skip task = _task(templates["alerts"], f"alerts-{case['id']}", env, bindings) - entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl"], "budget": _free()} + entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl", "summary.json"], "budget": _free()} entry["serialize"] = "engine" return fixed, task, entry diff --git a/src/openultrasast/plane/memory.py b/src/openultrasast/plane/memory.py index c620661..5c8df05 100644 --- a/src/openultrasast/plane/memory.py +++ b/src/openultrasast/plane/memory.py @@ -1,7 +1,7 @@ """The plane's memory store (harnessx-removal Requirement 6.1/6.4, design section 4). One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows -of five kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``proposal_outcome``) that the ``remember`` task +of six kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``proposal_outcome``) that ``remember`` (:mod:`.tasks.remember`) derives from a run's delivered artifacts. Layout, identical for both backends:: index.jsonl one row per ingested (run, task): sha256 of its rows, row count @@ -49,7 +49,7 @@ MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key" POPULATION_ANNOTATION = "openultrasast.io/population" SPLIT_ANNOTATION = "openultrasast.io/split" SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"} -KINDS = ("facts", "verdict", "unit_cost", "alert", "proposal_outcome") +KINDS = ("facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome") ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image") TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split") MIN_FREE_BYTES = 1 << 30 diff --git a/src/openultrasast/plane/tasks/alerts.py b/src/openultrasast/plane/tasks/alerts.py index f9916a4..5c31044 100644 --- a/src/openultrasast/plane/tasks/alerts.py +++ b/src/openultrasast/plane/tasks/alerts.py @@ -15,6 +15,15 @@ of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fix none). A fixed-pin alert inside the fix is a false alert on code the fix wrote: rule M1 of ``improve/memory.py`` counts it. +**Coverage** (an honest zero): quick mode has rules only for the languages its ruleset declares +(:func:`quick_languages`, derived from the bundled ruleset directory, never listed by hand). ``summary.json`` records +per language of the files the scan read whether quick mode covers it -- ``coverage: {language: {"coverage": "quick" | +"none", "files": {role: n}}}`` -- and ``uncovered``, the languages with ``none``. Zero alerts on an uncovered +language is not a clean result: nothing could have fired. ``remember`` turns the coverage into ``coverage`` rows, +and M1/M2 skip a repository and pin whose coverage for a rule's language is ``none``. The deterministic engine covers +more languages (:func:`engine_languages`, PHP among them); ``ousast plane alerts-engine`` (:mod:`..engine_alerts`) +produces those alerts on the host, where the engine image runs, and marks this task done. + Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_WORKSPACE_DIR`` (vulnerable checkout), ``OUSAST_FIXED_DIR`` (fixed checkout), ``OUSAST_INPUT_CASE`` (``case.json``), ``OUSAST_VULNERABLE_PIN``, ``OUSAST_FIXED_PIN``. A pin whose checkout yields no source file fails the task: an unread tree and a silent one are indistinguishable afterwards. Exit 0 done, 2 failed. @@ -26,7 +35,7 @@ import json import os import sys import traceback -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from pathlib import Path from typing import Any @@ -34,14 +43,55 @@ from ...findings import quick_scan_findings from ...mapping import analyze_entry_points, attach_reachability_hints from ...preprocess import detect_language, preprocess_repository from ...rank import rank_targets +from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset from .repo_facts import DECLARATION, GLOBAL, enclosing ROLES = ("vulnerable", "fixed") DIRS = ("OUSAST_WORKSPACE_DIR", "OUSAST_FIXED_DIR") PINS = ("OUSAST_VULNERABLE_PIN", "OUSAST_FIXED_PIN") +ANY_LANGUAGE = "*" # a rule that names no language runs on every file Ranges = Mapping[str, Sequence[Sequence[int]]] +def quick_languages(directory: Path = DEFAULT_RULESET_DIR) -> frozenset[str]: + """The languages quick mode has a rule for: the ``languages`` of every rule the ruleset directory holds that is + not disabled (``*`` when a rule names none: it runs on every file).""" + covered: set[str] = set() + for rule in load_ruleset(directory): + if rule.status != "disabled": + covered.update(rule.languages or (ANY_LANGUAGE,)) + return frozenset(covered) + + +def engine_languages(directory: Path = DEFAULT_RULESET_DIR) -> frozenset[str]: + """The languages the deterministic engine has a semantic model for: one ``semantic/.toml`` each.""" + return frozenset(path.stem for path in (directory / "semantic").glob("*.toml")) + + +def covers(covered: Iterable[str], language: str) -> bool: + languages = set(covered) + return ANY_LANGUAGE in languages or language in languages + + +def coverage(files: Mapping[str, Mapping[str, int]], quick: Iterable[str], engine: Iterable[str] = ()) -> dict[str, dict[str, Any]]: + """Per language of the files read (``{role: {language: n}}``): ``quick`` when quick mode has a rule for it, + ``engine`` when the engine's alerts were produced for it, else ``none``, with the file count per role.""" + quick, engine = set(quick), set(engine) + result: dict[str, dict[str, Any]] = {} + for role in sorted(files): + for language, number in files[role].items(): + if language == "unknown": + continue + kind = "quick" if covers(quick, language) else "engine" if language in engine else "none" + entry = result.setdefault(language, {"coverage": kind, "files": {}}) + entry["files"][role] = int(number) + return dict(sorted(result.items())) + + +def uncovered(cov: Mapping[str, Mapping[str, Any]]) -> list[str]: + return sorted(language for language, entry in cov.items() if entry.get("coverage") == "none") + + def _lines(path: Path) -> list[str]: try: return path.read_text(encoding="utf-8", errors="replace").splitlines() @@ -55,24 +105,35 @@ def in_range(ranges: Ranges | None, path: str, line: int | None) -> bool | None: return line is not None and any(int(lo) <= line <= int(hi) for lo, hi in ranges.get(path, [])) -def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, int]]: - """(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line.""" +def function_at(root: Path, path: str, line: int | None, fallback: str | None, cache: dict[str, list[str]]) -> str: + """The enclosing declaration of ``path:line`` by the ``repo-facts`` patterns, else ``fallback`` (````).""" + lines = cache.setdefault(path, _lines(root / path)) + language = detect_language(root / path) + if language in DECLARATION and line and 0 < line <= len(lines): + return enclosing(lines, line - 1, language) + return fallback or GLOBAL + + +def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line; what was read + counts files and bytes, and files per language.""" _, targets = preprocess_repository(root) targets = attach_reachability_hints(targets, analyze_entry_points(root, targets)) findings = quick_scan_findings(root, targets, rank_targets(targets)) - read = {"files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file())} + languages: dict[str, int] = {} + for target in targets: + languages[target.language] = languages.get(target.language, 0) + 1 + read = { + "files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file()), + "languages": dict(sorted(languages.items())), + } # fmt: skip cache: dict[str, list[str]] = {} rows: dict[tuple[str, str, int | None], dict[str, Any]] = {} for finding in findings: rule_id = finding.finding_id.split(":", 1)[0] - lines = cache.setdefault(finding.path, _lines(root / finding.path)) - language = detect_language(root / finding.path) - if language in DECLARATION and finding.line and 0 < finding.line <= len(lines): - function = enclosing(lines, finding.line - 1, language) - else: - function = finding.function_name or GLOBAL row = { - "rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line, "function": function, + "rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line, + "function": function_at(root, finding.path, finding.line, finding.function_name, cache), "pin_role": role, "pin": pin, "in_fix_range": in_range(ranges, finding.path, finding.line), } # fmt: skip rows.setdefault((rule_id, finding.path, finding.line), row) @@ -80,10 +141,14 @@ def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[d return ordered, read -def _summary(output_dir: Path, summary: Mapping[str, Any]) -> None: +def write_summary(output_dir: Path, summary: Mapping[str, Any]) -> None: (output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") +def write_rows(output_dir: Path, rows: Iterable[Mapping[str, Any]]) -> None: + (output_dir / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8") + + def main(environ: Mapping[str, str] | None = None) -> int: env = environ if environ is not None else os.environ output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None @@ -95,6 +160,7 @@ def main(environ: Mapping[str, str] | None = None) -> int: case = json.loads(Path(env["OUSAST_INPUT_CASE"]).read_text(encoding="utf-8")) if env.get("OUSAST_INPUT_CASE") else {} ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")} rows: list[dict[str, Any]] = [] + files: dict[str, dict[str, int]] = {} for role, directory, pin in zip(ROLES, DIRS, PINS, strict=True): root = Path(env.get(directory) or "") if not env.get(directory) or not root.is_dir(): @@ -102,21 +168,27 @@ def main(environ: Mapping[str, str] | None = None) -> int: found, read = scan(root, role, str(env.get(pin) or ""), ranges[role]) if read["files"] == 0: raise ValueError(f"the {role} checkout {root} has no source file the scan reads: an unread tree is not a clean one") + files[role] = read.pop("languages") summary.setdefault("read", {})[role] = read summary.setdefault("alerts", {})[role] = len(found) + summary.setdefault("pins", {})[role] = str(env.get(pin) or "") rows.extend(found) summary["units_done"] += 1 - (output / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8") + quick = quick_languages() + summary["quick_languages"] = sorted(quick) + summary["coverage"] = coverage(files, quick) + summary["uncovered"] = uncovered(summary["coverage"]) + write_rows(output, rows) summary["status"] = "done" except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback summary["reason"] = traceback.format_exc()[-2000:] if output is not None: - _summary(output, summary) - print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read")}), file=sys.stderr) + write_summary(output, summary) + print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read", "uncovered")}), file=sys.stderr) return 0 if summary["status"] == "done" else 2 -__all__ = ["in_range", "main", "scan"] +__all__ = ["coverage", "covers", "engine_languages", "function_at", "in_range", "main", "quick_languages", "scan", "uncovered"] if __name__ == "__main__": diff --git a/src/openultrasast/plane/tasks/loop.py b/src/openultrasast/plane/tasks/loop.py index 3fe87e6..3e49030 100644 --- a/src/openultrasast/plane/tasks/loop.py +++ b/src/openultrasast/plane/tasks/loop.py @@ -119,6 +119,10 @@ def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: tally_a = alerts.setdefault(str(a.get("rule_id")), {"vulnerable": 0, "fixed": 0, "fixed_in_fix_range": 0}) tally_a[role] += 1 tally_a["fixed_in_fix_range"] += role == "fixed" and a.get("in_fix_range") is True + coverage: dict[str, dict[str, int]] = {} # language -> pins per coverage kind: a `none` pin's zero alerts mean nothing + for c in (r for r in rows if r["kind"] == "coverage"): + tally_c = coverage.setdefault(str(c.get("language")), {"quick": 0, "engine": 0, "none": 0}) + tally_c[str(c.get("coverage"))] = tally_c.get(str(c.get("coverage")), 0) + 1 return { "cases": len({(v["repo"], v["pin"]) for v in verdicts}), "candidates": len(verdicts), @@ -130,6 +134,7 @@ def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: "usd_per_candidate": None if usd is None or not verdicts else round(usd / len(verdicts), 6), "dispute_rate": {f: {"candidates": n, "disputed": d, "rate": round(d / n, 4)} for f, (n, d) in sorted(families.items())}, "alerts": dict(sorted(alerts.items())), + "coverage": dict(sorted(coverage.items())), } diff --git a/src/openultrasast/plane/tasks/remember.py b/src/openultrasast/plane/tasks/remember.py index dd66d8e..867b89b 100644 --- a/src/openultrasast/plane/tasks/remember.py +++ b/src/openultrasast/plane/tasks/remember.py @@ -2,8 +2,9 @@ No model, budget ``{usd: 0, calls: 0}``, one per case. It reads the case's `facts.json`, the `units.jsonl` of the verify passes a, b and c, the final `agreed.json` (its per-candidate rows and `disputed` list) and, when present, -`alerts.jsonl`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`, `unit_cost` and `alert` (the -fifth kind, `proposal_outcome`, comes from the loop) -- and passes `facts.json` through. Task code interprets the +`alerts.jsonl` with the alerts' `summary.json`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`, +`unit_cost`, `alert` and `coverage` (per language and pin: whether any rule could have fired there; the kind +`proposal_outcome` comes from the loop) -- and passes `facts.json` through. Task code interprets the artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Req 3.4). Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`, @@ -12,7 +13,8 @@ Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS` (and `OUSAST_INPUT_FACTS_SUMMARY`, whose reused or recorded `image` the facts row carries; without one named, the row is omitted), `OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes' -`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound +`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS` (and `OUSAST_INPUT_ALERTS_SUMMARY`, +the coverage); the repository and pin from the bound Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from `OUSAST_POPULATION`/`OUSAST_SPLIT` (the generator copies the Run's `openultrasast.io/population` and `openultrasast.io/split` annotations). Exit 0 done, 2 failed. @@ -74,6 +76,26 @@ def _final(row: Mapping[str, Any], disputed: set[str]) -> str: return "disputed" if str(row["candidate"]) in disputed else "rejected" +def coverage_rows(ctx: Context, alerts_summary: Mapping[str, Any] | None) -> list[dict[str, Any]]: + """One ``coverage`` row per language and pin of the alerts' ``summary.json``: ``quick``, ``engine`` or ``none`` + (no rule could fire, so no alert there is evidence of anything), with the files read in that language. A summary + from before coverage was recorded yields none: unknown stays unknown.""" + if not alerts_summary or not isinstance(alerts_summary.get("coverage"), Mapping): + return [] + recorded = alerts_summary.get("pins") + pins: Mapping[str, Any] = recorded if isinstance(recorded, Mapping) else {} + rows: list[dict[str, Any]] = [] + for language, entry in sorted(alerts_summary["coverage"].items()): + for role, files in sorted((entry.get("files") or {}).items()): + fields: dict[str, Any] = {"language": language, "coverage": entry.get("coverage"), "files": files, "pin_role": role} + if pins.get(role): + fields["pin"] = str(pins[role]) + if alerts_summary.get("source"): + fields["source"] = str(alerts_summary["source"]) + rows.append(ctx.row("coverage", f"{language}:{role}", **fields)) + return rows + + def rows_for( ctx: Context, *, @@ -82,8 +104,9 @@ def rows_for( models: Mapping[str, str | None], agreed: Mapping[str, Any] | None, alerts: Sequence[Mapping[str, Any]] = (), + alerts_summary: Mapping[str, Any] | None = None, ) -> list[dict[str, Any]]: - """The memory rows of one case: facts, verdicts, unit costs, alerts.""" + """The memory rows of one case: facts, verdicts, unit costs, alerts, and the alerts' coverage per language and pin.""" rows: list[dict[str, Any]] = [] if facts is not None: loaded = json.loads(facts) @@ -115,8 +138,11 @@ def rows_for( fields["in_fix_range"] = alert["in_fix_range"] if alert.get("pin"): # a fixed-pin alert is a row of the pin it was raised on, not of the case's vulnerable pin fields["pin"] = str(alert["pin"]) + if alert.get("source"): # `engine` when `ousast plane alerts-engine` produced it on the host + fields["source"] = str(alert["source"]) subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "") rows.append(ctx.row("alert", subject, **fields)) + rows.extend(coverage_rows(ctx, alerts_summary)) return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows] @@ -185,7 +211,9 @@ def case_rows(base: Path, case: str, *, population: str, split: str) -> tuple[li passes = {p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES} models = {p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES} alerts = _jsonl(base / f"{case}-alerts" / "alerts.jsonl") - return rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts), facts + alerts_summary = _json(base / f"{case}-alerts" / "summary.json") + rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary) + return rows, facts def remember_run(run_dir: Path, store: MemoryStore, *, population: str = UNKNOWN, split: str = UNKNOWN) -> list[IngestResult]: @@ -246,7 +274,8 @@ def main(environ: Mapping[str, str] | None = None) -> int: passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES} models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES} agreed = _json(_path(env, "OUSAST_INPUT_AGREED")) - rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS"))) + alerts, alerts_summary = _jsonl(_path(env, "OUSAST_INPUT_ALERTS")), _json(_path(env, "OUSAST_INPUT_ALERTS_SUMMARY")) + rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary) facts_image = _facts_image(env, ctx.image) if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split) @@ -265,7 +294,7 @@ def main(environ: Mapping[str, str] | None = None) -> int: return 0 if summary["status"] == "done" else 2 -__all__ = ["Context", "candidates_digest", "case_rows", "context_from", "main", "remember_run", "rows_for"] +__all__ = ["Context", "candidates_digest", "case_rows", "context_from", "coverage_rows", "main", "remember_run", "rows_for"] if __name__ == "__main__": diff --git a/tests/fixtures/plane-engine/app/index.php b/tests/fixtures/plane-engine/app/index.php new file mode 100644 index 0000000..98b8fd0 --- /dev/null +++ b/tests/fixtures/plane-engine/app/index.php @@ -0,0 +1,5 @@ +", + "family": "injection", + "rung": "model_entailed", + "witness": "$_GET[\"id\"] -> mysqli_query($conn, \"SELECT * FROM users WHERE id = \" . $id) in app/index.php (line 3, 4 steps)", + "reached_from": 1 + }, + { + "site": "app/index.php:5:", + "family": "injection", + "rung": "model_entailed", + "witness": "$_GET[\"host\"] -> system(\"ping \" . $_GET[\"host\"]) in app/index.php (line 5, 2 steps)", + "reached_from": 1 + } + ] +} diff --git a/tests/fixtures/plane-engine/probe.stdout.txt b/tests/fixtures/plane-engine/probe.stdout.txt new file mode 100644 index 0000000..5a3dcf6 --- /dev/null +++ b/tests/fixtures/plane-engine/probe.stdout.txt @@ -0,0 +1,2 @@ +read 1 files, 145 bytes; 1 regions offer ('injection',) +{"root": "/case", "families": ["injection"], "seconds": 58.6, "questions": 1, "completed": 1, "degradations": [], "findings": 2} diff --git a/tests/test_improve_memory.py b/tests/test_improve_memory.py index 2a458c5..e079345 100644 --- a/tests/test_improve_memory.py +++ b/tests/test_improve_memory.py @@ -149,6 +149,34 @@ def test_m2_stays_silent_on_a_fixed_pin_alert_or_rejected_candidates() -> None: assert propose([*site_hits(), *rejected], rules) == [] +# ---- coverage: no rule could fire, so no evidence either way ------------------------------------------------------- + + +def covered(repo: str, pin: str, kind: str, language: str = "python", role: str = "vulnerable") -> dict[str, Any]: + return _row("coverage", repo, pin, f"{language}:{role}:{kind}", language=language, coverage=kind, files=3, pin_role=role) + + +def test_m1_skips_a_repository_and_pin_whose_coverage_for_the_rules_language_is_none() -> None: + blind = [*false_alerts(), covered("r2", P["a"], "none")] + assert propose(blind, [_rule("noisy")]) == [], "r2's alert could not be evidence: one repository is left" + wider = [*false_alerts(repos=("r1", "r1", "r2", "r3")), covered("r3", P["a"], "none")] + [proposal] = propose(wider, [_rule("noisy")]) + assert proposal.provenance["counts"] == {"false_alerts": 3, "repos": 2, "agreed_hits": 0} + assert proposal.provenance["excluded"]["coverage_none"] == {f"github.com/o/r3@{P['a']}": 1} + + +def test_coverage_none_of_another_language_or_beside_engine_coverage_is_no_skip() -> None: + other = [*false_alerts(), covered("r2", P["a"], "none", language="php")] + assert len(propose(other, [_rule("noisy")])) == 1, "the rule's language (python) was covered there" + engine = [*false_alerts(), covered("r2", P["a"], "none"), covered("r2", P["a"], "engine")] + assert len(propose(engine, [_rule("noisy")])) == 1, "a later run covered the pin: its alerts are evidence" + + +def test_m2_skips_a_blind_repository_and_pin() -> None: + rules = [_rule("quiet", status="shadow"), _rule("other")] + assert propose([*site_hits(), covered("r2", P["b"], "none")], rules) == [] + + # ---- M3: advisory only --------------------------------------------------------------------------------------------- diff --git a/tests/test_plane_engine_alerts.py b/tests/test_plane_engine_alerts.py new file mode 100644 index 0000000..b57bde6 --- /dev/null +++ b/tests/test_plane_engine_alerts.py @@ -0,0 +1,160 @@ +"""`ousast plane alerts-engine` (harnessx-removal Req 6.3): PHP alerts from the engine image, produced on the host. + +No container runs here: the docker runner is injected. The engine's output is the recorded probe +(``fixtures/plane-engine/probe.json``: 1 file read, 1 question asked and completed, 2 injection findings).""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import pytest +from test_plane_loop import LOOP, _templates, _write + +from openultrasast import cli +from openultrasast.plane import engine_alerts +from openultrasast.plane.engine_alerts import EngineAlertsError, alerts_engine, engine_rows, parse_site, read_record +from openultrasast.plane.generate import CaseInputs, render + +FIXTURE = Path(__file__).parent / "fixtures" / "plane-engine" +CASE = "php-probe" +RECORD = json.loads((FIXTURE / "probe.json").read_text()) +LOG = (FIXTURE / "probe.stdout.txt").read_text() + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=True).stdout.strip() + + +@pytest.fixture +def world(tmp_path: Path) -> dict[str, Any]: + """A case cache with a vulnerable and a fixed commit, a Run with the case's `alerts` task, a frozen source.""" + repo = tmp_path / "cache" / CASE + (repo / "app").mkdir(parents=True) + _git(repo, "init", "-q") + shutil.copy(FIXTURE / "app" / "index.php", repo / "app" / "index.php") + _git(repo, "add", ".") + _git(repo, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "vulnerable") + vulnerable = _git(repo, "rev-parse", "HEAD") + text = (repo / "app" / "index.php").read_text().replace('" . $id', '" . intval($id)') + (repo / "app" / "index.php").write_text(text) + _git(repo, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qam", "fixed") + fixed = _git(repo, "rev-parse", "HEAD") + record = {"id": CASE, "repo": "https://example.com/reserved/probe", "family": "injection", "vulnerable": vulnerable, "fixed": fixed} + candidate = ("app/index.php", "", 3) + case = CaseInputs( + record, (candidate,), (candidate,), {}, 0.0, ranges={"app/index.php": [(3, 3)]}, fixed_ranges={"app/index.php": [(3, 3)]} + ) + files = render([case], _templates(), "validation-46", "t", population="p", split="s", loop=LOOP) + source = tmp_path / "frozen" + (source / "src" / "openultrasast").mkdir(parents=True) + (source / "benchmarks" / "push").mkdir(parents=True) + (source / "benchmarks" / "push" / "finding_dump.py").write_text("") + run = _write(tmp_path / "plane", files) + return {"run": run, "source": source, "cache": tmp_path / "cache", "pins": (vulnerable, fixed), "results": tmp_path / "results"} + + +class FakeDocker: + """Answers `docker image inspect` and writes ``record`` as the engine's result for every `docker run`.""" + + def __init__(self, record: dict[str, Any] | None = RECORD, log: str = LOG) -> None: + self.record, self.log, self.runs = record, log, [] # type: ignore[var-annotated] + + def __call__(self, command: Sequence[str], timeout: float) -> subprocess.CompletedProcess[str]: + command = list(command) + if command[:3] == ["docker", "image", "inspect"]: + return subprocess.CompletedProcess(command, 0, "sha256:feed\n", "") + mounts = [command[i + 1] for i, part in enumerate(command) if part == "-v"] + checkout = Path(next(m for m in mounts if m.endswith(":/case:ro")).split(":")[0]) + assert (checkout / "app" / "index.php").is_file(), "the pin was exported before the container ran" + self.runs.append({"command": command, "timeout": timeout}) + if self.record is not None: + out = Path(next(m for m in mounts if m.endswith(":/out")).split(":")[0]) + (out / command[command.index("--out") + 1].removeprefix("/out/")).write_text(json.dumps(self.record)) + return subprocess.CompletedProcess(command, 0, self.log, "") + + +def _produce(world: dict[str, Any], docker: FakeDocker, tmp_path: Path) -> list[dict[str, Any]]: + return alerts_engine( + world["run"], source=world["source"], work=tmp_path / "work", cache=world["cache"], deadline=120, runner=docker, + results_root=world["results"], + ) # fmt: skip + + +def test_recorded_engine_findings_become_alert_rows() -> None: + assert parse_site("app/index.php:3:") == ("app/index.php", 3, "") + assert parse_site("lib/A.php:76:Foo::bar") == ("lib/A.php", 76, "Foo::bar") + record = read_record(FIXTURE / "probe.json", LOG, 0) + assert (record["files"], record["bytes"], record["questions"], record["completed"]) == (1, 145, 1, 1) + rows = engine_rows(record, FIXTURE, "vulnerable", "a" * 40, {"app/index.php": [[3, 3]]}) + assert rows == [ + {"rule_id": "engine:injection", "rule_status": "enabled", "path": "app/index.php", "line": line, "function": "", + "pin_role": "vulnerable", "pin": "a" * 40, "in_fix_range": line == 3, "source": "engine"} + for line in (3, 5) + ] # fmt: skip + + +def test_an_engine_that_read_nothing_or_asked_nothing_fails(tmp_path: Path) -> None: + out = tmp_path / "x.json" + with pytest.raises(EngineAlertsError, match="wrote no result"): + read_record(out, "boom", 125) + out.write_text(json.dumps({**RECORD, "questions": 0})) + with pytest.raises(EngineAlertsError, match="no question was asked"): + read_record(out, LOG, 0) + out.write_text(json.dumps(RECORD)) + with pytest.raises(EngineAlertsError, match="did not report reading"): + read_record(out, "", 0) + + +def test_alerts_engine_writes_the_alerts_task_outputs_and_marks_it_done(world: dict[str, Any], tmp_path: Path) -> None: + docker = FakeDocker() + [result] = _produce(world, docker, tmp_path) + assert result["status"] == "done" and len(docker.runs) == 2, "one container per pin, one after the other" + for run in docker.runs: + command = run["command"] + assert command[command.index("--memory") + 1] == "3g" and command[command.index("--network") + 1] == "none" + assert command[command.index("--families") + 1] == "injection" and run["timeout"] == 120 + engine_alerts.GRACE + out = world["results"] / "validation-46" / f"{CASE}-alerts" + rows = [json.loads(line) for line in (out / "alerts.jsonl").read_text().splitlines()] + vulnerable, fixed = world["pins"] + got = [(r["pin_role"], r["pin"], r["line"], r["in_fix_range"], r["source"]) for r in rows] + assert got == [("vulnerable", vulnerable, 3, True, "engine"), ("vulnerable", vulnerable, 5, False, "engine"), + ("fixed", fixed, 3, True, "engine"), ("fixed", fixed, 5, False, "engine")] # fmt: skip + summary = json.loads((out / "summary.json").read_text()) + assert (summary["status"], summary["units_done"], summary["units_total"], summary["source"]) == ("done", 2, 2, "engine") + assert summary["engine"]["vulnerable"] == { + "files": 1, "bytes": 145, "questions": 1, "completed": 1, "seconds": 58.6, "degradations": [], "exit": 0, "findings": 2 + } # fmt: skip + assert summary["coverage"] == {"php": {"coverage": "engine", "files": {"fixed": 1, "vulnerable": 1}}} and summary["uncovered"] == [] + assert summary["in_fix_range"] == {"vulnerable": 1, "fixed": 1} and summary["image"] == "sha256:feed" + assert isinstance(summary["seconds"], float) + state = json.loads((world["results"] / "validation-46" / "state.json").read_text())["tasks"][f"{CASE}-alerts"] + assert state["status"] == "done" and state["reused"]["source"] == "engine", "the reconciler skips it like reused facts" + assert not list((tmp_path / "work").glob(f"{CASE}--*/")) + + +def test_a_failed_engine_leaves_the_run_untouched(world: dict[str, Any], tmp_path: Path) -> None: + with pytest.raises(EngineAlertsError, match="no question was asked"): + _produce(world, FakeDocker({**RECORD, "questions": 0, "findings": []}), tmp_path) + with pytest.raises(EngineAlertsError, match="wrote no result"): + _produce(world, FakeDocker(None), tmp_path) + assert not (world["results"] / "validation-46").exists(), "no alerts written, nothing marked done" + + +def test_the_source_must_be_a_frozen_export(world: dict[str, Any], tmp_path: Path) -> None: + (world["source"] / ".git").mkdir() + with pytest.raises(EngineAlertsError, match="export it first"): + _produce(world, FakeDocker(), tmp_path) + + +def test_the_cli_reports_and_fails_loudly(world: dict[str, Any], tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: # fmt: skip + monkeypatch.setattr(engine_alerts, "run_command", FakeDocker({**RECORD, "questions": 0})) + argv = ["plane", "alerts-engine", str(world["run"]), "--source", str(world["source"]), "--repos", str(world["cache"])] + monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "r")) + assert cli.main(argv) == 2 + assert "no question was asked" in capsys.readouterr().err diff --git a/tests/test_plane_loop.py b/tests/test_plane_loop.py index 0fdadd5..6840a53 100644 --- a/tests/test_plane_loop.py +++ b/tests/test_plane_loop.py @@ -97,6 +97,39 @@ def test_alerts_attribute_functions_and_mark_the_fix_on_both_pins(tmp_path: Path assert "an unread tree is not a clean one" in json.loads((out / "summary.json").read_text())["reason"] +def test_quick_coverage_is_derived_from_the_ruleset_directory(tmp_path: Path) -> None: + shipped = alerts.quick_languages() + assert {"python", "javascript", "typescript", "java", "c", "groovy"} <= shipped and "php" not in shipped + assert "php" in alerts.engine_languages(), "the engine's semantic models cover PHP" + rules = tmp_path / "ruleset" / "php" + rules.mkdir(parents=True) + rule = 'rule_id = "php-x"\ntitle = "x"\nlanguages = ["php"]\ncwe = "CWE-89"\ntags = []\npattern = "x"\n' + (rules / "rules.toml").write_text(f"[[rule]]\n{rule}") + assert alerts.quick_languages(tmp_path / "ruleset") == {"php"}, "a ruleset added for a language covers it" + (rules / "rules.toml").write_text(f'[[rule]]\n{rule}status = "disabled"\n') + assert alerts.quick_languages(tmp_path / "ruleset") == set(), "a disabled rule fires nowhere" + + +def test_alerts_on_an_uncovered_language_record_coverage_none_not_a_clean_zero(tmp_path: Path) -> None: + for side in ("v", "f"): + (tmp_path / side).mkdir() + (tmp_path / side / "index.php").write_text(' None: repo = tmp_path / "r" repo.mkdir() @@ -129,6 +162,7 @@ def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path) names = [e.name for e in run.tasks] assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS] assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl" + assert run.task(f"{CASE}-remember").inputs["alerts_summary"] == f"{CASE}-alerts/summary.json", "the coverage travels too" measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"} assert run.task("loop-measure").inputs == measure_inputs assert run.task("loop-improve").producers == ("loop-propose",) diff --git a/tests/test_plane_remember.py b/tests/test_plane_remember.py index 4af5b94..3c8a8cc 100644 --- a/tests/test_plane_remember.py +++ b/tests/test_plane_remember.py @@ -99,6 +99,38 @@ def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None: assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic" +COVERAGE_SUMMARY = { + "status": "done", "pins": {"vulnerable": "0" * 40, "fixed": "f" * 40}, + "coverage": { + "javascript": {"coverage": "quick", "files": {"vulnerable": 2}}, + "php": {"coverage": "none", "files": {"vulnerable": 9, "fixed": 9}}, + }, +} # fmt: skip + + +def test_the_alerts_coverage_becomes_one_row_per_language_and_pin() -> None: + ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64) + rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts=[], alerts_summary=COVERAGE_SUMMARY) + got = sorted((r["language"], r["pin_role"], r["pin"], r["coverage"], r["files"]) for r in rows if r["kind"] == "coverage") + assert got == [ + ("javascript", "vulnerable", "0" * 40, "quick", 2), ("php", "fixed", "f" * 40, "none", 9), + ("php", "vulnerable", "0" * 40, "none", 9), + ] # fmt: skip + assert rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts_summary={"status": "done"}) == [], "unknown stays unknown" + + +def test_remember_run_carries_the_alerts_coverage_into_the_store(run_dir: Path, tmp_path: Path) -> None: + (run_dir / f"{CASE}-alerts").mkdir() + (run_dir / f"{CASE}-alerts" / "alerts.jsonl").write_text("") # zero alerts: meaningful only through the coverage + summary = {**COVERAGE_SUMMARY, "pins": {"vulnerable": PIN, "fixed": "f" * 40}} + (run_dir / f"{CASE}-alerts" / "summary.json").write_text(json.dumps(summary)) + store = FileStore(tmp_path / "memory") + [result] = remember_run(run_dir, store) + assert result.kinds["coverage"] == 3 + php = sorted((r["pin"], r["coverage"]) for r in _by_kind(store)["coverage"] if r["language"] == "php") + assert php == [(PIN, "none"), ("f" * 40, "none")] + + def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None: docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text())) task = next(d for d in docs if d["kind"] == "Task")