marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(plane): honest coverage in alerts; PHP alerts from the engine on the host

Browse Source 
Quick mode has rules only for c/cpp, groovy, java, javascript/typescript and
python, so every PHP case's "0 alerts" read as clean. The `alerts` task now
records per language of the files read whether quick mode covers it
(derived from the bundled ruleset, not a list): coverage quick | none, with
file counts per pin, and `uncovered`. `remember` turns that into `coverage`
memory rows (a new kind; the Run passes the alerts summary.json as input),
loop-measure reports coverage per language, and improve/memory.py's M1/M2
skip a repository+pin whose coverage for the rule's language is only
`none` (counted under `coverage_none` in the proposal's excluded).

`ousast plane alerts-engine <Run.yaml> --source <frozen export>`
(plane/engine_alerts.py) does the alerts task on the host for cases whose
files include a language the engine covers but quick mode does not (PHP):
exports both pins from the case cache, runs the quick scan and the
openultrasast:dev engine container (finding_dump.py, --network none,
--memory 3g, one at a time, per-pin deadline), writes engine findings in
the alerts.jsonl row shape (rule_id engine:<family>, source engine,
in_fix_range from case.json) with a summary (files read, questions,
completed, seconds, degradations per pin), and marks the task done via
reconciler.mark_done. No result, no files read, or questions == 0 fails
loudly and leaves the run untouched. Refuses a git checkout as source.

Runner image rebuilt and re-pinned (sha256:ebb40a9a...); validation-46
regenerated (digest only). Live, recorded in
benchmarks/measurements/2026-09-30-php-engine-alerts/: froxlor 366 files,
130/130 questions per pin, 6 alerts per pin, 0 in the fix range; wp-gopay
19 files, 101/101 questions, 0 alerts. The loop: 32 run rows (coverage 8,
php=engine), 0 proposals, gate no_proposals, $0.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 13:44:32 +02:00
co-authored by Claude Opus 5.5
parent f4bd984fda
commit 45d02ecf20
35 changed files with 1949 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/measurements/2026-09-08-module-audit.json
+14 -3
View File
@@ -1,7 +1,7 @@
{
"modules": 134,
"modules": 135,
"counts": {
"load_bearing": 127,
"load_bearing": 128,
"standalone_capability": 7
},
"orphaned": [],
@@ -662,6 +662,7 @@
"cli",
"improve.memory",
"plane.egress",
"plane.engine_alerts",
"plane.generate",
"plane.memory",
"plane.reconciler",
@@ -698,6 +699,12 @@
"plane.reconciler"
]
},
{
"module": "plane.engine_alerts",
"classification": "load_bearing",
"reason": "harnessx-removal Req 6.3: engine alerts where quick mode has no rules (PHP), on the host",
"importers": []
},
{
"module": "plane.generate",
"classification": "load_bearing",
@@ -761,6 +768,7 @@
"reason": "ai-service-plane Req 4.1: the task entrypoint package the runner imports by name; runner integration pending (task 3)",
"importers": [
"cli",
"plane.engine_alerts",
"plane.generate",
"plane.memory",
"plane.tasks.agree",
@@ -778,7 +786,9 @@
"module": "plane.tasks.alerts",
"classification": "load_bearing",
"reason": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free",
"importers": []
"importers": [
"plane.engine_alerts"
]
},
{
"module": "plane.tasks.loop",
@@ -1139,6 +1149,7 @@
"improve.memory",
"improve.validator",
"pairs",
"plane.tasks.alerts",
"plane.tasks.loop",
"stage_processors"
]
benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts-summary.json
+96
View File
@@ -0,0 +1,96 @@
{
"alerts": {
"fixed": 6,
"vulnerable": 6
},
"alerts_engine": {
"fixed": 6,
"vulnerable": 6
},
"calls": 0,
"coverage": {
"javascript": {
"coverage": "quick",
"files": {
"fixed": 18,
"vulnerable": 18
}
},
"php": {
"coverage": "engine",
"files": {
"fixed": 348,
"vulnerable": 348
}
}
},
"deadline": 1800.0,
"engine": {
"fixed": {
"bytes": 4256572,
"completed": 130,
"degradations": [
"cross_partition_semantics_unresolved"
],
"exit": 0,
"files": 366,
"findings": 6,
"questions": 130,
"seconds": 383.2
},
"vulnerable": {
"bytes": 4256094,
"completed": 130,
"degradations": [
"cross_partition_semantics_unresolved"
],
"exit": 0,
"files": 366,
"findings": 6,
"questions": 130,
"seconds": 398.2
}
},
"engine_languages": [
"php"
],
"family": "injection",
"host": true,
"image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"in_fix_range": {
"fixed": 0,
"vulnerable": 0
},
"model": null,
"pins": {
"fixed": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e",
"vulnerable": "9bf03de643a2edee4642bfc6c94db1f05253da02"
},
"quick_languages": [
"c",
"cpp",
"groovy",
"java",
"javascript",
"python",
"typescript"
],
"read": {
"fixed": {
"bytes": 4256572,
"files": 366
},
"vulnerable": {
"bytes": 4256094,
"files": 366
}
},
"seconds": 789.1,
"source": "engine",
"status": "done",
"uncovered": [],
"units_done": 2,
"units_total": 2,
"usage": {},
"usd": 0
}
benchmarks/measurements/2026-09-30-php-engine-alerts/froxlor-admin-ip-second-order-sqli-alerts.jsonl
+12
View File
@@ -0,0 +1,12 @@
{"function": "<global>", "in_fix_range": false, "line": 109, "path": "admin_configfiles.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "lockJob", "in_fix_range": false, "line": 232, "path": "lib/Froxlor/Cli/MasterCron.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "safe_exec", "in_fix_range": false, "line": 258, "path": "lib/Froxlor/FileDir.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "safe_exec", "in_fix_range": false, "line": 260, "path": "lib/Froxlor/FileDir.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "checkDatabase", "in_fix_range": false, "line": 364, "path": "lib/Froxlor/Install/Install.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "checkDatabase", "in_fix_range": false, "line": 367, "path": "lib/Froxlor/Install/Install.php", "pin": "9bf03de643a2edee4642bfc6c94db1f05253da02", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "<global>", "in_fix_range": false, "line": 109, "path": "admin_configfiles.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "lockJob", "in_fix_range": false, "line": 232, "path": "lib/Froxlor/Cli/MasterCron.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "safe_exec", "in_fix_range": false, "line": 258, "path": "lib/Froxlor/FileDir.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "safe_exec", "in_fix_range": false, "line": 260, "path": "lib/Froxlor/FileDir.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "checkDatabase", "in_fix_range": false, "line": 364, "path": "lib/Froxlor/Install/Install.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
{"function": "checkDatabase", "in_fix_range": false, "line": 367, "path": "lib/Froxlor/Install/Install.php", "pin": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e", "pin_role": "fixed", "rule_id": "engine:injection", "rule_status": "enabled", "source": "engine"}
benchmarks/measurements/2026-09-30-php-engine-alerts/gate.json
+29
View File
@@ -0,0 +1,29 @@
{
"accepted": false,
"adoption": null,
"benchmark_edits": [],
"manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml",
"memory_edits": [],
"outcome": {
"accepted": false,
"degradations": [],
"edits": [],
"fp_after": 0.0,
"fp_before": 0.0,
"matched_after": 2,
"matched_before": 2,
"per_profile_after": {},
"per_profile_before": {},
"profile_regressions": [],
"profiles_under_minimum": [],
"reason": "no_proposals",
"recall_after": 1.0,
"recall_before": 1.0,
"round": 1,
"score_after": 73,
"score_before": 73
},
"proposals": 0,
"reason": "no_proposals",
"round": 1
}
benchmarks/measurements/2026-09-30-php-engine-alerts/measure.json
+170
View File
@@ -0,0 +1,170 @@
{
"inputs_bytes": {
"froxlor_admin_ip_second_order_sqli": 15137,
"wp_gopay_log_filter_sqli": 4027
},
"rows": {
"run": 32,
"snapshot": 191,
"union": 211
},
"run": "validation-46",
"run_metrics": {
"agreed": 3,
"alerts": {
"engine:injection": {
"fixed": 6,
"fixed_in_fix_range": 0,
"vulnerable": 6
}
},
"candidates": 4,
"cases": 2,
"coverage": {
"javascript": {
"engine": 0,
"none": 0,
"quick": 4
},
"php": {
"engine": 4,
"none": 0,
"quick": 0
}
},
"declared_sites_agreed": 3,
"declared_sites_matched": 4,
"dispute_rate": {
"injection": {
"candidates": 4,
"disputed": 1,
"rate": 0.25
}
},
"disputed": 1,
"rejected": 0,
"tiebreaks": 3,
"usd": 0.10796,
"usd_per_candidate": 0.02699
},
"store_metrics": {
"agreed": 28,
"alerts": {
"c-shell-exec": {
"fixed": 2,
"fixed_in_fix_range": 0,
"vulnerable": 2
},
"c-unsafe-memory-copy": {
"fixed": 2,
"fixed_in_fix_range": 0,
"vulnerable": 2
},
"python-shell-true": {
"fixed": 1,
"fixed_in_fix_range": 0,
"vulnerable": 1
},
"python-unsafe-deserialization": {
"fixed": 8,
"fixed_in_fix_range": 0,
"vulnerable": 8
},
"python-unsafe-eval": {
"fixed": 23,
"fixed_in_fix_range": 0,
"vulnerable": 23
}
},
"candidates": 43,
"cases": 14,
"coverage": {},
"declared_sites_agreed": 16,
"declared_sites_matched": 19,
"dispute_rate": {
"injection": {
"candidates": 38,
"disputed": 5,
"rate": 0.1316
},
"untrusted_destination": {
"candidates": 5,
"disputed": 0,
"rate": 0.0
}
},
"disputed": 5,
"rejected": 10,
"tiebreaks": 10,
"usd": 0.856679,
"usd_per_candidate": 0.019923
},
"union_metrics": {
"agreed": 28,
"alerts": {
"c-shell-exec": {
"fixed": 2,
"fixed_in_fix_range": 0,
"vulnerable": 2
},
"c-unsafe-memory-copy": {
"fixed": 2,
"fixed_in_fix_range": 0,
"vulnerable": 2
},
"engine:injection": {
"fixed": 6,
"fixed_in_fix_range": 0,
"vulnerable": 6
},
"python-shell-true": {
"fixed": 1,
"fixed_in_fix_range": 0,
"vulnerable": 1
},
"python-unsafe-deserialization": {
"fixed": 8,
"fixed_in_fix_range": 0,
"vulnerable": 8
},
"python-unsafe-eval": {
"fixed": 23,
"fixed_in_fix_range": 0,
"vulnerable": 23
}
},
"candidates": 43,
"cases": 14,
"coverage": {
"javascript": {
"engine": 0,
"none": 0,
"quick": 4
},
"php": {
"engine": 4,
"none": 0,
"quick": 0
}
},
"declared_sites_agreed": 16,
"declared_sites_matched": 19,
"dispute_rate": {
"injection": {
"candidates": 38,
"disputed": 5,
"rate": 0.1316
},
"untrusted_destination": {
"candidates": 5,
"disputed": 0,
"rate": 0.0
}
},
"disputed": 5,
"rejected": 10,
"tiebreaks": 10,
"usd": 0.856679,
"usd_per_candidate": 0.019923
}
}
benchmarks/measurements/2026-09-30-php-engine-alerts/record.json
+391
View File
@@ -0,0 +1,391 @@
{
"engine_image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"host_resources_during_engine_runs": {
"disk_free_mb_min": 2102,
"engine_container_gib_peak": 2.129,
"host_available_mb_min": 1952,
"host_total_mb": 7877,
"host_used_mb_peak": 5924,
"samples": 61
},
"new_spend_usd": 0.0,
"note": "Engine alerts for the two PHP loop cases of validation-46, produced on the host by `ousast plane alerts-engine` (openultrasast:dev, frozen export of f4bd984's src + finding_dump.py, --network none --memory 3g, one container at a time, deadline 1800 s per pin), then the loop's remember -> measure -> propose -> improve on the local ax with the rebuilt runner image. The quick-mode alerts these cases had before (0 per pin) were a zero on an uncovered language.",
"runner_image": "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8",
"tasks": {
"froxlor-admin-ip-second-order-sqli-alerts": {
"finished": "2026-09-30T11:32:02+00:00",
"model": null,
"reason": null,
"reused": {
"alerts": {
"fixed": 6,
"vulnerable": 6
},
"host": true,
"image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"seconds": 789.1,
"source": "engine"
},
"started": "2026-09-30T09:50:49+00:00",
"status": "done",
"summary": {
"alerts": {
"fixed": 6,
"vulnerable": 6
},
"alerts_engine": {
"fixed": 6,
"vulnerable": 6
},
"calls": 0,
"coverage": {
"javascript": {
"coverage": "quick",
"files": {
"fixed": 18,
"vulnerable": 18
}
},
"php": {
"coverage": "engine",
"files": {
"fixed": 348,
"vulnerable": 348
}
}
},
"deadline": 1800.0,
"engine": {
"fixed": {
"bytes": 4256572,
"completed": 130,
"degradations": [
"cross_partition_semantics_unresolved"
],
"exit": 0,
"files": 366,
"findings": 6,
"questions": 130,
"seconds": 383.2
},
"vulnerable": {
"bytes": 4256094,
"completed": 130,
"degradations": [
"cross_partition_semantics_unresolved"
],
"exit": 0,
"files": 366,
"findings": 6,
"questions": 130,
"seconds": 398.2
}
},
"engine_languages": [
"php"
],
"family": "injection",
"host": true,
"image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"in_fix_range": {
"fixed": 0,
"vulnerable": 0
},
"model": null,
"pins": {
"fixed": "a1eaca5a1601c8a30e00814a4fc73ad0c185f89e",
"vulnerable": "9bf03de643a2edee4642bfc6c94db1f05253da02"
},
"quick_languages": [
"c",
"cpp",
"groovy",
"java",
"javascript",
"python",
"typescript"
],
"read": {
"fixed": {
"bytes": 4256572,
"files": 366
},
"vulnerable": {
"bytes": 4256094,
"files": 366
}
},
"seconds": 789.1,
"source": "engine",
"status": "done",
"uncovered": [],
"units_done": 2,
"units_total": 2,
"usage": {},
"usd": 0
},
"usd": null
},
"froxlor-admin-ip-second-order-sqli-remember": {
"finished": "2026-09-30T11:38:01+00:00",
"model": null,
"reason": null,
"started": "2026-09-30T11:37:32+00:00",
"status": "done",
"summary": {
"calls": 0,
"facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it",
"model": null,
"rows": 25,
"status": "done",
"units_done": 1,
"units_total": 1,
"usage": {},
"usd": 0
},
"usd": 0
},
"loop-improve": {
"finished": "2026-09-30T11:39:55+00:00",
"model": null,
"reason": null,
"started": "2026-09-30T11:39:24+00:00",
"status": "done",
"summary": {
"accepted": false,
"calls": 0,
"edits": 0,
"memory_edits": 0,
"model": null,
"reason": "no_proposals",
"status": "done",
"step": "improve",
"units_done": 1,
"units_total": 1,
"usage": {},
"usd": 0
},
"usd": 0
},
"loop-measure": {
"finished": "2026-09-30T11:38:54+00:00",
"model": null,
"reason": null,
"started": "2026-09-30T11:38:28+00:00",
"status": "done",
"summary": {
"calls": 0,
"model": null,
"rows": {
"run": 32,
"snapshot": 191,
"union": 211
},
"status": "done",
"step": "measure",
"units_done": 1,
"units_total": 1,
"usage": {},
"usd": 0
},
"usd": 0
},
"loop-propose": {
"finished": "2026-09-30T11:39:24+00:00",
"model": null,
"reason": null,
"started": "2026-09-30T11:38:54+00:00",
"status": "done",
"summary": {
"calls": 0,
"excluded": 0,
"kept": 211,
"model": null,
"proposals": 0,
"rows": 211,
"status": "done",
"step": "propose",
"units_done": 1,
"units_total": 1,
"usage": {},
"usd": 0
},
"usd": 0
},
"memory-snapshot": {
"finished": "2026-09-30T11:37:31+00:00",
"model": null,
"reused": {
"index_digest": "5e9299a551eea21289224ee063b2eb8cdf3f845a69967baf8b8c02d5be9cfc74",
"kept": 191,
"rows": 191,
"snapshot": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory"
},
"started": null,
"status": "done",
"summary": {
"calls": 0,
"model": null,
"reused": {
"excluded": {
"gated_case": {},
"holdout_pair": {},
"population": {},
"rows": 0
},
"guard": {
"catalog": "benchmarks/pairs/catalog.toml",
"manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml",
"populations": []
},
"index_digest": "5e9299a551eea21289224ee063b2eb8cdf3f845a69967baf8b8c02d5be9cfc74",
"index_entries": 19,
"kept": 191,
"rows": 191,
"store": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory"
},
"status": "done",
"units_done": 1,
"units_total": 1,
"usage": {},
"usd": null
},
"usd": null
},
"wp-gopay-log-filter-sqli-alerts": {
"finished": "2026-09-30T11:36:07+00:00",
"model": null,
"reason": null,
"reused": {
"alerts": {
"fixed": 0,
"vulnerable": 0
},
"host": true,
"image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"seconds": 245.2,
"source": "engine"
},
"started": "2026-09-30T09:52:27+00:00",
"status": "done",
"summary": {
"alerts": {
"fixed": 0,
"vulnerable": 0
},
"alerts_engine": {
"fixed": 0,
"vulnerable": 0
},
"calls": 0,
"coverage": {
"javascript": {
"coverage": "quick",
"files": {
"fixed": 4,
"vulnerable": 4
}
},
"php": {
"coverage": "engine",
"files": {
"fixed": 15,
"vulnerable": 15
}
}
},
"deadline": 1800.0,
"engine": {
"fixed": {
"bytes": 150539,
"completed": 101,
"degradations": [
"cross_partition_semantics_unresolved",
"vendor_semantics_unresolved"
],
"exit": 0,
"files": 19,
"findings": 0,
"questions": 101,
"seconds": 122.0
},
"vulnerable": {
"bytes": 150528,
"completed": 101,
"degradations": [
"cross_partition_semantics_unresolved",
"vendor_semantics_unresolved"
],
"exit": 0,
"files": 19,
"findings": 0,
"questions": 101,
"seconds": 120.2
}
},
"engine_languages": [
"php"
],
"family": "injection",
"host": true,
"image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"in_fix_range": {
"fixed": 0,
"vulnerable": 0
},
"model": null,
"pins": {
"fixed": "7190c985370f9758f226c2f2addd79532a2f93d5",
"vulnerable": "626bc8b5f8d33de6eefd921e3a05ed52ac320bcd"
},
"quick_languages": [
"c",
"cpp",
"groovy",
"java",
"javascript",
"python",
"typescript"
],
"read": {
"fixed": {
"bytes": 150539,
"files": 19
},
"vulnerable": {
"bytes": 150528,
"files": 19
}
},
"seconds": 245.2,
"source": "engine",
"status": "done",
"uncovered": [],
"units_done": 2,
"units_total": 2,
"usage": {},
"usd": 0
},
"usd": null
},
"wp-gopay-log-filter-sqli-remember": {
"finished": "2026-09-30T11:38:28+00:00",
"model": null,
"reason": null,
"started": "2026-09-30T11:38:01+00:00",
"status": "done",
"summary": {
"calls": 0,
"facts_row": "omitted: the facts task's summary names no image, and a reuse key must not guess it",
"model": null,
"rows": 7,
"status": "done",
"units_done": 1,
"units_total": 1,
"usage": {},
"usd": 0
},
"usd": 0
}
}
}
benchmarks/measurements/2026-09-30-php-engine-alerts/run.yaml
+235
View File
@@ -0,0 +1,235 @@
# generated by `ousast plane workspaces benchmarks/independent/population-v2.toml --validation-set /tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/phpgap/set/validation.json --candidates ~/ousast-results/independent-v2-modelsinks --triage ~/ousast-results/independent-v2-batched-check/scans --repos ~/.cache/openultrasast/independent --run validation-46 --loop --loop-commit 1ad472c68d759e4ef010bd7a2e23532b1de538a9 --loop-repo https://github.com/norandom/OpenUltraSAST.git --loop-manifest benchmarks/manifests/java-spring-boot-vulnerable.toml`; do not edit, regenerate.
# 2 cases, 4 candidates (4 kept by the recorded triage); usd ceiling 0.27.
apiVersion: openultrasast.io/v1alpha1
kind: Run
metadata:
name: validation-46
annotations:
openultrasast.io/population: population-v2
openultrasast.io/split: validation
spec:
tasks:
- name: froxlor-admin-ip-second-order-sqli-facts
task: repo-facts-froxlor-admin-ip-second-order-sqli
outputs:
- facts.json
budget: &id001
usd: 0
calls: 0
serialize: repo-facts
- name: froxlor-admin-ip-second-order-sqli-va
task: verify-a-froxlor-admin-ip-second-order-sqli
inputs:
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
outputs:
- units.jsonl
- summary.json
budget:
usd: 0.07
calls: 16
- name: froxlor-admin-ip-second-order-sqli-vb
task: verify-b-froxlor-admin-ip-second-order-sqli
inputs:
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
outputs:
- units.jsonl
- summary.json
budget:
usd: 0.07
calls: 16
- name: froxlor-admin-ip-second-order-sqli-agree
task: agree-froxlor-admin-ip-second-order-sqli
inputs:
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
outputs:
- agreed.json
- disputed.json
budget: *id001
- name: wp-gopay-log-filter-sqli-facts
task: repo-facts-wp-gopay-log-filter-sqli
outputs:
- facts.json
budget: &id002
usd: 0
calls: 0
serialize: repo-facts
- name: wp-gopay-log-filter-sqli-va
task: verify-a-wp-gopay-log-filter-sqli
inputs:
facts: wp-gopay-log-filter-sqli-facts/facts.json
outputs:
- units.jsonl
- summary.json
budget:
usd: 0.02
calls: 8
- name: wp-gopay-log-filter-sqli-vb
task: verify-b-wp-gopay-log-filter-sqli
inputs:
facts: wp-gopay-log-filter-sqli-facts/facts.json
outputs:
- units.jsonl
- summary.json
budget:
usd: 0.02
calls: 8
- name: wp-gopay-log-filter-sqli-agree
task: agree-wp-gopay-log-filter-sqli
inputs:
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
outputs:
- agreed.json
- disputed.json
budget: *id002
- name: froxlor-admin-ip-second-order-sqli-vc
task: verify-c-froxlor-admin-ip-second-order-sqli
dependsOn:
- froxlor-admin-ip-second-order-sqli-agree
inputs:
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
only: froxlor-admin-ip-second-order-sqli-agree/disputed.json
outputs:
- units.jsonl
- summary.json
budget:
usd: 0.07
calls: 16
- name: froxlor-admin-ip-second-order-sqli-final
task: agree-froxlor-admin-ip-second-order-sqli
inputs:
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl
outputs:
- agreed.json
- disputed.json
budget:
usd: 0
calls: 0
- name: wp-gopay-log-filter-sqli-vc
task: verify-c-wp-gopay-log-filter-sqli
dependsOn:
- wp-gopay-log-filter-sqli-agree
inputs:
facts: wp-gopay-log-filter-sqli-facts/facts.json
only: wp-gopay-log-filter-sqli-agree/disputed.json
outputs:
- units.jsonl
- summary.json
budget:
usd: 0.02
calls: 8
- name: wp-gopay-log-filter-sqli-final
task: agree-wp-gopay-log-filter-sqli
inputs:
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl
outputs:
- agreed.json
- disputed.json
budget:
usd: 0
calls: 0
- name: froxlor-admin-ip-second-order-sqli-alerts
task: alerts-froxlor-admin-ip-second-order-sqli
outputs:
- alerts.jsonl
- summary.json
budget:
usd: 0
calls: 0
serialize: engine
- name: froxlor-admin-ip-second-order-sqli-remember
task: remember-froxlor-admin-ip-second-order-sqli
inputs:
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
facts_summary: froxlor-admin-ip-second-order-sqli-facts/summary.json
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
summary_a: froxlor-admin-ip-second-order-sqli-va/summary.json
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
summary_b: froxlor-admin-ip-second-order-sqli-vb/summary.json
pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl
summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json
agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json
alerts: froxlor-admin-ip-second-order-sqli-alerts/alerts.jsonl
alerts_summary: froxlor-admin-ip-second-order-sqli-alerts/summary.json
outputs:
- memory.jsonl
budget:
usd: 0
calls: 0
- name: wp-gopay-log-filter-sqli-alerts
task: alerts-wp-gopay-log-filter-sqli
outputs:
- alerts.jsonl
- summary.json
budget:
usd: 0
calls: 0
serialize: engine
- name: wp-gopay-log-filter-sqli-remember
task: remember-wp-gopay-log-filter-sqli
inputs:
facts: wp-gopay-log-filter-sqli-facts/facts.json
facts_summary: wp-gopay-log-filter-sqli-facts/summary.json
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
summary_a: wp-gopay-log-filter-sqli-va/summary.json
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
summary_b: wp-gopay-log-filter-sqli-vb/summary.json
pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl
summary_c: wp-gopay-log-filter-sqli-vc/summary.json
agreed: wp-gopay-log-filter-sqli-final/agreed.json
alerts: wp-gopay-log-filter-sqli-alerts/alerts.jsonl
alerts_summary: wp-gopay-log-filter-sqli-alerts/summary.json
outputs:
- memory.jsonl
budget:
usd: 0
calls: 0
- name: memory-snapshot
task: memory-snapshot
outputs:
- rows.jsonl
- index.json
budget:
usd: 0
calls: 0
- name: loop-measure
task: loop-measure
inputs:
remember_froxlor-admin-ip-second-order-sqli: froxlor-admin-ip-second-order-sqli-remember/memory.jsonl
remember_wp-gopay-log-filter-sqli: wp-gopay-log-filter-sqli-remember/memory.jsonl
snapshot: memory-snapshot/rows.jsonl
outputs:
- measure.json
- rows.jsonl
budget:
usd: 0
calls: 0
- name: loop-propose
task: loop-propose
inputs:
measure: loop-measure/measure.json
rows: loop-measure/rows.jsonl
index: memory-snapshot/index.json
outputs:
- proposals.jsonl
- memory_proposals.jsonl
- signals.json
budget:
usd: 0
calls: 0
- name: loop-improve
task: loop-improve
inputs:
proposals: loop-propose/proposals.jsonl
outputs:
- gate.json
- journal.json
budget:
usd: 0
calls: 0
serialize: engine
benchmarks/measurements/2026-09-30-php-engine-alerts/set.json
+22
View File
@@ -0,0 +1,22 @@
{
"wp-gopay-log-filter-sqli": [
[
"admin/views/log.php",
"<global>"
]
],
"froxlor-admin-ip-second-order-sqli": [
[
"lib/Froxlor/Api/Commands/Domains.php",
"validateIpAddresses"
],
[
"lib/Froxlor/Api/Commands/IpsAndPorts.php",
"listing"
],
[
"lib/Froxlor/Api/Commands/IpsAndPorts.php",
"listingCount"
]
]
}
benchmarks/measurements/2026-09-30-php-engine-alerts/signals.json
+8
View File
@@ -0,0 +1,8 @@
{
"advisory": true,
"disputed_families": [],
"thresholds": {
"min_candidates": 5,
"min_dispute_rate": 0.3
}
}
benchmarks/measurements/2026-09-30-php-engine-alerts/snapshot-index.json
+18
View File
@@ -0,0 +1,18 @@
{
"excluded": {
"gated_case": {},
"holdout_pair": {},
"population": {},
"rows": 0
},
"guard": {
"catalog": "benchmarks/pairs/catalog.toml",
"manifest": "benchmarks/manifests/java-spring-boot-vulnerable.toml",
"populations": []
},
"index_digest": "5e9299a551eea21289224ee063b2eb8cdf3f845a69967baf8b8c02d5be9cfc74",
"index_entries": 19,
"kept": 191,
"rows": 191,
"store": "file:///tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/hxr2/memory"
}
benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts-summary.json
+98
View File
@@ -0,0 +1,98 @@
{
"alerts": {
"fixed": 0,
"vulnerable": 0
},
"alerts_engine": {
"fixed": 0,
"vulnerable": 0
},
"calls": 0,
"coverage": {
"javascript": {
"coverage": "quick",
"files": {
"fixed": 4,
"vulnerable": 4
}
},
"php": {
"coverage": "engine",
"files": {
"fixed": 15,
"vulnerable": 15
}
}
},
"deadline": 1800.0,
"engine": {
"fixed": {
"bytes": 150539,
"completed": 101,
"degradations": [
"cross_partition_semantics_unresolved",
"vendor_semantics_unresolved"
],
"exit": 0,
"files": 19,
"findings": 0,
"questions": 101,
"seconds": 122.0
},
"vulnerable": {
"bytes": 150528,
"completed": 101,
"degradations": [
"cross_partition_semantics_unresolved",
"vendor_semantics_unresolved"
],
"exit": 0,
"files": 19,
"findings": 0,
"questions": 101,
"seconds": 120.2
}
},
"engine_languages": [
"php"
],
"family": "injection",
"host": true,
"image": "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb",
"in_fix_range": {
"fixed": 0,
"vulnerable": 0
},
"model": null,
"pins": {
"fixed": "7190c985370f9758f226c2f2addd79532a2f93d5",
"vulnerable": "626bc8b5f8d33de6eefd921e3a05ed52ac320bcd"
},
"quick_languages": [
"c",
"cpp",
"groovy",
"java",
"javascript",
"python",
"typescript"
],
"read": {
"fixed": {
"bytes": 150539,
"files": 19
},
"vulnerable": {
"bytes": 150528,
"files": 19
}
},
"seconds": 245.2,
"source": "engine",
"status": "done",
"uncovered": [],
"units_done": 2,
"units_total": 2,
"usage": {},
"usd": 0
}
benchmarks/measurements/2026-09-30-php-engine-alerts/wp-gopay-log-filter-sqli-alerts.jsonl
View File
plane/tasks/agree.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ metadata:
name: agree
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9"
image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8"
command: ["agree"]
resources:
requests:
plane/tasks/alerts.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ metadata:
name: alerts
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9"
image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8"
command: ["alerts"]
resources:
requests:
plane/tasks/loop.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ metadata:
name: loop
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9"
image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8"
command: ["loop"]
resources:
requests:
plane/tasks/remember.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ metadata:
name: remember
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9"
image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8"
command: ["remember"]
resources:
requests:
plane/tasks/repo-facts.yaml
+1 -1
View File
@@ -8,7 +8,7 @@ metadata:
name: repo-facts
atespace: default
spec:
image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9"
image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8"
command: ["repo-facts"]
resources:
requests:
plane/tasks/validation-46.yaml
+105 -105
View File
@@ -5,9 +5,9 @@ metadata:
name: repo-facts-budibase-mongo-template-nosqli
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}'
openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -36,7 +36,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -67,7 +67,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -96,7 +96,7 @@ metadata:
name: agree-budibase-mongo-template-nosqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -123,9 +123,9 @@ metadata:
name: repo-facts-cct-studio-server-cmdi
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}'
openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -154,7 +154,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -185,7 +185,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -214,7 +214,7 @@ metadata:
name: agree-cct-studio-server-cmdi
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -241,9 +241,9 @@ metadata:
name: repo-facts-contextforge-prompt-ssti
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}'
openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -272,7 +272,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -303,7 +303,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -332,7 +332,7 @@ metadata:
name: agree-contextforge-prompt-ssti
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -359,9 +359,9 @@ metadata:
name: repo-facts-froxlor-admin-ip-second-order-sqli
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}'
openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -390,7 +390,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -421,7 +421,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -450,7 +450,7 @@ metadata:
name: agree-froxlor-admin-ip-second-order-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -477,9 +477,9 @@ metadata:
name: repo-facts-fuxa-tdengine-escape-sqli
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}'
openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -508,7 +508,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -539,7 +539,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -568,7 +568,7 @@ metadata:
name: agree-fuxa-tdengine-escape-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -595,9 +595,9 @@ metadata:
name: repo-facts-lmdeploy-media-url-ssrf
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}'
openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -626,7 +626,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -657,7 +657,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -686,7 +686,7 @@ metadata:
name: agree-lmdeploy-media-url-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -713,9 +713,9 @@ metadata:
name: repo-facts-medplum-external-auth-open-redirect
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}'
openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -744,7 +744,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -775,7 +775,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -804,7 +804,7 @@ metadata:
name: agree-medplum-external-auth-open-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -831,9 +831,9 @@ metadata:
name: repo-facts-pgadmin-maintenance-sqli
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}'
openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -862,7 +862,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -893,7 +893,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -922,7 +922,7 @@ metadata:
name: agree-pgadmin-maintenance-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -949,9 +949,9 @@ metadata:
name: repo-facts-qwed-sympy-eval
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}'
openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -980,7 +980,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1011,7 +1011,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1040,7 +1040,7 @@ metadata:
name: agree-qwed-sympy-eval
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1067,9 +1067,9 @@ metadata:
name: repo-facts-reactpress-findall-column-sqli
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}'
openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -1098,7 +1098,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1129,7 +1129,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1158,7 +1158,7 @@ metadata:
name: agree-reactpress-findall-column-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1185,9 +1185,9 @@ metadata:
name: repo-facts-unleash-addon-webhook-ssrf
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}'
openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -1216,7 +1216,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1247,7 +1247,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1276,7 +1276,7 @@ metadata:
name: agree-unleash-addon-webhook-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1303,9 +1303,9 @@ metadata:
name: repo-facts-winml-cli-serve-cors
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}'
openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -1334,7 +1334,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1365,7 +1365,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1394,7 +1394,7 @@ metadata:
name: agree-winml-cli-serve-cors
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1421,9 +1421,9 @@ metadata:
name: repo-facts-wp-directorist-avatar-ssrf
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}'
openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -1452,7 +1452,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1483,7 +1483,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1512,7 +1512,7 @@ metadata:
name: agree-wp-directorist-avatar-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1539,9 +1539,9 @@ metadata:
name: repo-facts-wp-gopay-log-filter-sqli
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}'
openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -1570,7 +1570,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1601,7 +1601,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1630,7 +1630,7 @@ metadata:
name: agree-wp-gopay-log-filter-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1657,9 +1657,9 @@ metadata:
name: repo-facts-wp-groundhogg-confirm-redirect
atespace: default
annotations:
openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}'
openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}'
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- repo-facts
env:
@@ -1688,7 +1688,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1719,7 +1719,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1748,7 +1748,7 @@ metadata:
name: agree-wp-groundhogg-confirm-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- agree
env:
@@ -1777,7 +1777,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1808,7 +1808,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1839,7 +1839,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1870,7 +1870,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1901,7 +1901,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1932,7 +1932,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1963,7 +1963,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -1994,7 +1994,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2025,7 +2025,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2056,7 +2056,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2087,7 +2087,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2118,7 +2118,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2149,7 +2149,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2180,7 +2180,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2211,7 +2211,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- verify
env:
@@ -2240,7 +2240,7 @@ metadata:
name: remember-budibase-mongo-template-nosqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2265,7 +2265,7 @@ metadata:
name: remember-cct-studio-server-cmdi
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2290,7 +2290,7 @@ metadata:
name: remember-contextforge-prompt-ssti
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2315,7 +2315,7 @@ metadata:
name: remember-froxlor-admin-ip-second-order-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2340,7 +2340,7 @@ metadata:
name: remember-fuxa-tdengine-escape-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2365,7 +2365,7 @@ metadata:
name: remember-lmdeploy-media-url-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2390,7 +2390,7 @@ metadata:
name: remember-medplum-external-auth-open-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2415,7 +2415,7 @@ metadata:
name: remember-pgadmin-maintenance-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2440,7 +2440,7 @@ metadata:
name: remember-qwed-sympy-eval
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2465,7 +2465,7 @@ metadata:
name: remember-reactpress-findall-column-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2490,7 +2490,7 @@ metadata:
name: remember-unleash-addon-webhook-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2515,7 +2515,7 @@ metadata:
name: remember-winml-cli-serve-cors
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2540,7 +2540,7 @@ metadata:
name: remember-wp-directorist-avatar-ssrf
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2565,7 +2565,7 @@ metadata:
name: remember-wp-gopay-log-filter-sqli
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
@@ -2590,7 +2590,7 @@ metadata:
name: remember-wp-groundhogg-confirm-redirect
atespace: default
spec:
image: localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9
image: localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8
command:
- remember
env:
plane/tasks/verify.yaml
+1 -1
View File
@@ -10,7 +10,7 @@ metadata:
annotations:
openultrasast.io/model: deepseek-flash
spec:
image: "localhost:5001/ousast-runner@sha256:7fbd523f127f1e15eddc48b06e73acf75fa70f741d36e0749e74ca883797f6c9"
image: "localhost:5001/ousast-runner@sha256:ebb40a9a0985bb0665a74cbaf3db5ab0c31efda369a5f5366c51f7c12fad68c8"
command: ["verify"]
resources:
requests:
src/openultrasast/cli.py
+32
View File
@@ -274,6 +274,18 @@ def _main(argv: list[str] | None) -> int:
plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/<run>.yaml names population and split")
plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)")
plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)")
plane_engine = plane_sub.add_parser(
"alerts-engine", help="a Run's `alerts` for languages quick mode does not cover (PHP), from the engine image on this host"
)
plane_engine.add_argument("run_manifest", type=Path)
plane_engine.add_argument(
"--source", type=Path, required=True, help="a frozen export: git archive HEAD src benchmarks/push/finding_dump.py"
)
plane_engine.add_argument("--cases", default="", help="comma-separated case ids (default: every alerts case of the Run)")
plane_engine.add_argument("--deadline", type=float, default=1800.0, help="the engine's scan deadline per pin, seconds")
plane_engine.add_argument("--image", default="openultrasast:dev")
plane_engine.add_argument("--repos", type=Path, help="the case cache (default ~/.cache/openultrasast/independent)")
plane_engine.add_argument("--work", type=Path, help="where pins are exported and results land (default: a temporary directory)")
plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
plane_ws.add_argument("population", type=Path)
plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
@@ -1318,6 +1330,8 @@ def _plane(args: argparse.Namespace) -> int:
if args.plane_command == "status":
print(reconciler.status(args.run, units=args.units))
return 0
if args.plane_command == "alerts-engine":
return _plane_alerts_engine(args)
if args.plane_command == "doctor":
checks = reconciler.doctor()
for name, ok, text in checks:
@@ -1332,6 +1346,24 @@ def _plane(args: argparse.Namespace) -> int:
return 0
def _plane_alerts_engine(args: argparse.Namespace) -> int:
from .plane import engine_alerts
only = [c.strip() for c in args.cases.split(",") if c.strip()]
try:
results = engine_alerts.alerts_engine(
args.run_manifest, source=args.source, only=only, work=args.work, cache=args.repos or engine_alerts.CACHE,
deadline=args.deadline, image=args.image,
) # fmt: skip
except (engine_alerts.EngineAlertsError, OSError, ValueError, subprocess.CalledProcessError) as exc:
print(f"alerts-engine failed: {exc}", file=sys.stderr)
return 2
keys = ("case", "status", "reason", "alerts", "alerts_engine", "in_fix_range", "engine", "uncovered", "seconds")
for result in results:
print(json.dumps({k: result[k] for k in keys if k in result}, sort_keys=True))
return 0
def _plane_memory(
step: str, run_manifest: Path | None, *, run: str | None = None, population: str | None = None, split: str | None = None
) -> int:
src/openultrasast/improve/memory.py
+32 -2
View File
@@ -19,6 +19,11 @@ final counts as no evidence in either direction.
- **M3, disputed families**: families with at least :data:`M3_MIN_CANDIDATES` candidates and a dispute rate of at
least :data:`M3_MIN_DISPUTE_RATE`. Advisory (``signals.json``); never an edit, never shown to the validator.
**Coverage**: M1 and M2 skip every repository and pin whose ``coverage`` rows record the rule's language only as
``none`` (quick mode had no rule for it and no engine alerts were produced): a rule cannot be demoted or promoted
on evidence that could not exist. The skipped alerts are counted per ``repo@pin`` under ``coverage_none`` in the
proposal's ``excluded``.
**Train-on-test guard** (:class:`Guard`), applied before any rule sees a row: a row is dropped when its population
is one the proposal will be qualified on, its (repo, pin) is a case of the gated benchmark manifest, or its
repository is a holdout pair's repository. The pair catalog records a holdout pair's *fix* commit, not the
@@ -142,6 +147,7 @@ class _View:
verdicts: dict[tuple[str, str, str], Record] = field(default_factory=dict) # (repo, pin, candidate)
sites: set[tuple[str, str]] = field(default_factory=set) # (repo, candidate) with site_match true
alerts: dict[str, list[Record]] = field(default_factory=dict) # rule_id -> alerts
coverage: dict[tuple[str, str, str], set[str]] = field(default_factory=dict) # (repo, pin, language) -> recorded kinds
@classmethod
def of(cls, records: Sequence[Record]) -> _View:
@@ -151,7 +157,9 @@ class _View:
seen: set[tuple[object, ...]] = set()
for rec in records:
row = rec.row
if row["kind"] == "verdict":
if row["kind"] == "coverage" and row.get("language"):
view.coverage.setdefault((row["repo"], row["pin"], str(row["language"])), set()).add(str(row.get("coverage")))
elif row["kind"] == "verdict":
view.verdicts[(row["repo"], row["pin"], str(row.get("candidate")))] = rec
if row.get("site_match") is True:
view.sites.add((row["repo"], str(row.get("candidate"))))
@@ -171,6 +179,26 @@ class _View:
flag = alert.get("in_fix_range")
return bool(flag) if flag is not None else (alert["repo"], _cand(alert)) in self.sites
def blind(self, repo: str, pin: str, languages: Iterable[str]) -> bool:
"""True when a language of the rule was recorded at this repository and pin with coverage ``none`` only: no
rule could fire there, so what the rule did or did not say there is no evidence either way."""
return any(self.coverage.get((repo, pin, language)) == {"none"} for language in languages)
def seeing(self, languages_by_rule: Mapping[str, Sequence[str]]) -> tuple[_View, dict[str, int]]:
"""This view without the alerts at a repository and pin blind to their rule's language, and how many were
dropped per ``repo@pin``."""
kept: dict[str, list[Record]] = {}
dropped: dict[str, int] = {}
for rule_id, alerts in self.alerts.items():
languages = languages_by_rule.get(rule_id, ())
for alert in alerts:
if self.blind(alert.row["repo"], alert.row["pin"], languages):
label = f"{alert.row['repo']}@{alert.row['pin']}"
dropped[label] = dropped.get(label, 0) + 1
else:
kept.setdefault(rule_id, []).append(alert)
return _View(self.verdicts, self.sites, kept, self.coverage), dict(sorted(dropped.items()))
def _m1(rule_id: str, view: _View) -> tuple[list[Record], dict[str, int]] | None:
evidence: list[Record] = []
@@ -262,7 +290,9 @@ def propose_from_memory(
) -> list[MemoryProposal]:
"""M1 and M2 over the rows the guard keeps, minus proposals already reverted on the same evidence."""
kept, dropped = excluded.apply(rows)
view = _View.of(kept)
view, blind = _View.of(kept).seeing({rid: rule.languages for rid, rule in ruleset_by_id.items()})
if blind:
dropped = {**dropped, "coverage_none": blind}
statuses = {rid: _status(rid, ruleset_by_id, current_ledger) for rid in ruleset_by_id}
enabled = {rid for rid, status in statuses.items() if status == "enabled"}
proposals: list[MemoryProposal] = []
src/openultrasast/model/audit.py
+1
View File
@@ -66,6 +66,7 @@ SPEC_OWNED = {
"plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`",
"plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free",
"plane.tasks.alerts": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free",
"plane.engine_alerts": "harnessx-removal Req 6.3: engine alerts where quick mode has no rules (PHP), on the host",
"plane.tasks.loop": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps",
"plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`",
"plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory",
src/openultrasast/plane/engine_alerts.py
+257
View File
@@ -0,0 +1,257 @@
"""Engine alerts for the languages quick mode does not cover, produced on the host (harnessx-removal Req 6.3).
The loop's ``alerts`` task runs the quick-mode scan, which has rules only for the languages its ruleset declares; a
PHP case yields zero alerts that mean nothing (``coverage: none``). PHP detection comes from the Joern taint engine
(``ruleset/semantic/php.toml``), which runs in the ``openultrasast:dev`` image, not in the runner sandbox. So
``ousast plane alerts-engine <Run.yaml>`` does the ``alerts`` task's work on the host for every case whose files
include a language quick mode does not cover but the engine does:
- export the case's vulnerable and fixed pins from the case cache (``git archive``, as
``benchmarks/independent/evaluate.py`` does), run the quick scan on each (:func:`.tasks.alerts.scan`) and the
engine container on each -- ``benchmarks/push/finding_dump.py`` from a FROZEN source export (never a live tree),
``--network none --memory 3g``, the case's family, one container at a time, a per-case deadline;
- convert the engine's findings into the ``alerts.jsonl`` row shape of the ``alerts`` task (``rule_id`` is
``engine:<family>``, ``source: "engine"``, ``in_fix_range`` from ``case.json``'s ranges), write both kinds of rows
and a ``summary.json`` (``coverage`` with the engine's languages as ``engine``, the engine's files read, questions,
completed, seconds and degradations per pin) into ``<run dir>/<case>-alerts/``;
- mark the task done with :func:`.reconciler.mark_done`, the mechanism fact reuse uses, so the Run skips it.
The instrument must have read its input: no result file, no ``read N files`` line, or ``questions == 0`` fails the
case loudly (:class:`EngineAlertsError`) and leaves the run directory untouched.
"""
from __future__ import annotations
import json
import re
import shutil
import subprocess
import tempfile
import time
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from . import reconciler
from .tasks.alerts import ROLES, coverage, covers, engine_languages, function_at, in_range, quick_languages, scan, uncovered
from .tasks.alerts import write_rows as write_alert_rows
from .tasks.alerts import write_summary as write_alert_summary
IMAGE = "openultrasast:dev"
MEMORY = "3g"
DEADLINE = 1800.0
GRACE = 600.0 # the container's own startup and teardown beyond the scan deadline
CACHE = Path.home() / ".cache" / "openultrasast" / "independent"
DUMP = Path("benchmarks") / "push" / "finding_dump.py"
_READ = re.compile(r"^read (\d+) files, (\d+) bytes", re.MULTILINE)
_SITE = re.compile(r"^(.+?):(\d+):(.*)$")
Runner = Callable[[Sequence[str], float], subprocess.CompletedProcess[str]]
class EngineAlertsError(RuntimeError):
"""The engine did not demonstrably read its input, or a case cannot be prepared."""
def run_command(command: Sequence[str], timeout: float) -> subprocess.CompletedProcess[str]:
return subprocess.run(list(command), capture_output=True, text=True, timeout=timeout, check=False)
@dataclass(frozen=True)
class EngineCase:
case_id: str
entry: str # the Run entry of the case's `alerts` task
repo: str
pins: Mapping[str, str] # role -> commit
family: str
ranges: Mapping[str, Any] # role -> {path: [[lo, hi], ...]} or None
def engine_cases(run_manifest: Path) -> tuple[str, list[EngineCase]]:
"""(Run name, every `alerts` entry of the Run as a case): pins from the task's env, family and fix ranges from the
`case.json` of its bound inputs Workspace, the repository from its bound git Workspace."""
run, manifests = reconciler.load_run(Path(run_manifest))
cases: list[EngineCase] = []
for entry in run.tasks:
task = manifests.tasks[entry.task]
if tuple(task.command[:1]) != ("alerts",):
continue
env = {e.name: e.value for e in task.env}
bound = [manifests.workspaces[b.name] for b in task.workspaces if b.name in manifests.workspaces]
files = {f.path: f.content for ws in bound for f in ws.files}
repos = [g.repo for ws in bound for g in ws.git]
if "case.json" not in files or not repos:
raise EngineAlertsError(f"{entry.name}: no bound case.json or git Workspace")
case = json.loads(files["case.json"])
pins = {"vulnerable": env.get("OUSAST_VULNERABLE_PIN", ""), "fixed": env.get("OUSAST_FIXED_PIN", "")}
if not all(pins.values()) or not case.get("family"):
raise EngineAlertsError(f"{entry.name}: pins {pins} or family {case.get('family')!r} missing")
case_id = str(case.get("id") or entry.name.removesuffix("-alerts"))
ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")}
cases.append(EngineCase(case_id, entry.name, repos[0], pins, str(case["family"]), ranges))
return run.metadata.name, cases
def frozen_source(source: Path) -> Path:
"""The analyzer tree the engine container mounts: an export (``git archive HEAD src benchmarks/push/finding_dump.py``),
never a live checkout, whose files could change under a long run."""
source = Path(source).resolve()
if (source / ".git").exists():
raise EngineAlertsError(f"{source} is a git checkout; export it first (git archive HEAD src {DUMP.as_posix()})")
if not (source / "src" / "openultrasast").is_dir() or not (source / DUMP).is_file():
raise EngineAlertsError(f"{source} holds no src/openultrasast and {DUMP.as_posix()}")
return source
def export_pin(cache_repo: Path, pin: str, target: Path) -> Path:
"""``git archive <pin>`` of the case cache into ``target`` (emptied first)."""
if target.exists():
shutil.rmtree(target)
target.mkdir(parents=True)
archive = target.parent / f"{target.name}.tar"
with archive.open("wb") as stream:
subprocess.run(["git", "-C", str(cache_repo), "archive", pin], stdout=stream, check=True)
subprocess.run(["tar", "-xf", str(archive), "-C", str(target)], check=True)
archive.unlink()
if not any(target.rglob("*")):
raise EngineAlertsError(f"{cache_repo} @ {pin}: the export is empty")
return target
def docker_command(source: Path, checkout: Path, out_dir: Path, name: str, family: str, deadline: float, image: str) -> list[str]:
return [
"docker", "run", "--rm", "--init", "--network", "none", "--memory", MEMORY, "--entrypoint", "python",
"-e", "OUSAST_LOG_LEVEL=WARNING", "-e", "PYTHONPATH=/new/src",
"-v", f"{source / 'src'}:/new/src:ro", "-v", f"{source / DUMP}:/dump.py:ro",
"-v", f"{checkout}:/case:ro", "-v", f"{out_dir}:/out",
image, "/dump.py", "--root", "/case", "--families", family,
"--out", f"/out/{name}.json", "--deadline", str(int(deadline)), "--max-regions", "5000",
] # fmt: skip
def read_record(out: Path, log: str, returncode: int) -> dict[str, Any]:
"""The engine's result, with the files and bytes it said it read; fails unless it read input and asked questions."""
tail = log.strip()[-600:]
if not out.is_file():
raise EngineAlertsError(f"the engine wrote no result {out.name} (exit {returncode}): {tail}")
record = json.loads(out.read_text(encoding="utf-8"))
read = _READ.search(log)
if read is None or int(read.group(1)) == 0:
raise EngineAlertsError(f"{out.name}: the engine did not report reading any file (exit {returncode}): {tail}")
if not record.get("questions"):
raise EngineAlertsError(f"{out.name}: no question was asked (exit {returncode}): the engine measured nothing")
return {**record, "files": int(read.group(1)), "bytes": int(read.group(2)), "exit": returncode}
def parse_site(site: str) -> tuple[str, int | None, str | None]:
match = _SITE.match(site)
if match is None:
return site, None, None
return match.group(1), int(match.group(2)), match.group(3) or None
def engine_rows(record: Mapping[str, Any], root: Path, role: str, pin: str, ranges: Mapping[str, Any] | None) -> list[dict[str, Any]]:
"""The engine's findings as ``alerts.jsonl`` rows (the ``alerts`` task's shape plus ``source: "engine"``), one per
rule, file, line; ``function`` by the ``repo-facts`` patterns when ``root`` holds the file, else the engine's."""
cache: dict[str, list[str]] = {}
rows: dict[tuple[str, str, int | None], dict[str, Any]] = {}
for finding in record.get("findings") or []:
path, line, function = parse_site(str(finding["site"]))
rule_id = f"engine:{finding['family']}"
row = {
"rule_id": rule_id, "rule_status": "enabled", "path": path, "line": line,
"function": function_at(root, path, line, function, cache), "pin_role": role, "pin": pin,
"in_fix_range": in_range(ranges, path, line), "source": "engine",
} # fmt: skip
rows.setdefault((rule_id, path, line), row)
return sorted(rows.values(), key=lambda r: (r["path"], r["line"] or 0, r["rule_id"]))
def _image_id(image: str, runner: Runner) -> str:
done = runner(["docker", "image", "inspect", "--format", "{{.Id}}", image], 60.0)
return done.stdout.strip() if done.returncode == 0 and done.stdout.strip() else image
def produce(
case: EngineCase, run_name: str, source: Path, work: Path, *, cache: Path = CACHE, deadline: float = DEADLINE,
image: str = IMAGE, runner: Runner | None = None, results_root: Path | None = None,
) -> dict[str, Any]: # fmt: skip
"""Quick and engine alerts of one case on both pins into the run directory, the task marked done; the summary.
A case whose files are all in quick-covered languages is ``skipped`` and nothing is written."""
runner = runner or run_command
started = time.monotonic()
quick, engine = quick_languages(), engine_languages()
rows: list[dict[str, Any]] = []
files: dict[str, dict[str, int]] = {}
summary: dict[str, Any] = {"read": {}, "alerts": {}, "alerts_engine": {}, "in_fix_range": {}, "engine": {}, "pins": dict(case.pins)}
needed: set[str] = set()
for role in ROLES:
checkout = export_pin(cache / case.case_id, case.pins[role], work / f"{case.case_id}--{role}")
try:
quick_rows, read = scan(checkout, role, case.pins[role], case.ranges.get(role))
files[role] = read.pop("languages")
if read["files"] == 0:
raise EngineAlertsError(f"{case.case_id} {role}: the quick scan read no file of {checkout}")
needed |= {lang for lang in files[role] if lang != "unknown" and not covers(quick, lang) and lang in engine}
if not needed:
return {"case": case.case_id, "status": "skipped", "reason": "quick mode covers every language read", "languages": files}
out_dir = work / "out"
out_dir.mkdir(parents=True, exist_ok=True)
name = f"{case.case_id}--{role}"
(out_dir / f"{name}.json").unlink(missing_ok=True)
command = docker_command(source, checkout, out_dir, name, case.family, deadline, image)
try:
done = runner(command, deadline + GRACE)
except subprocess.TimeoutExpired as exc:
raise EngineAlertsError(f"{name}: the engine container outlived {deadline + GRACE:g}s") from exc
record = read_record(out_dir / f"{name}.json", (done.stdout or "") + "\n" + (done.stderr or ""), done.returncode)
found = engine_rows(record, checkout, role, case.pins[role], case.ranges.get(role))
finally:
shutil.rmtree(checkout, ignore_errors=True)
rows.extend(quick_rows + found)
summary["read"][role] = read
summary["alerts"][role] = len(quick_rows) + len(found)
summary["alerts_engine"][role] = len(found)
summary["in_fix_range"][role] = sum(1 for r in found if r["in_fix_range"] is True)
keys = ("files", "bytes", "questions", "completed", "seconds", "degradations", "exit")
summary["engine"][role] = {**{k: record.get(k) for k in keys}, "findings": len(record.get("findings") or [])}
cov = coverage(files, quick, needed)
summary.update(
status="done", units_done=len(ROLES), units_total=len(ROLES), usd=0, calls=0, usage={}, model=None, source="engine",
host=True, family=case.family, image=_image_id(image, runner), deadline=deadline, quick_languages=sorted(quick),
engine_languages=sorted(needed), coverage=cov, uncovered=uncovered(cov), seconds=round(time.monotonic() - started, 1),
) # fmt: skip
output = reconciler.run_dir(run_name, results_root) / case.entry
output.mkdir(parents=True, exist_ok=True)
write_alert_rows(output, rows)
write_alert_summary(output, summary)
reused = {"source": "engine", "host": True, "image": summary["image"], "alerts": summary["alerts"], "seconds": summary["seconds"]}
reconciler.mark_done(run_name, case.entry, reused=reused, results_root=results_root)
return {"case": case.case_id, **summary}
def alerts_engine(
run_manifest: Path, *, source: Path, only: Sequence[str] = (), work: Path | None = None, cache: Path = CACHE,
deadline: float = DEADLINE, image: str = IMAGE, runner: Runner | None = None, results_root: Path | None = None,
) -> list[dict[str, Any]]: # fmt: skip
""":func:`produce` for every `alerts` case of the Run (or those named), one engine container at a time."""
run_name, cases = engine_cases(run_manifest)
unknown = set(only) - {c.case_id for c in cases}
if unknown:
raise EngineAlertsError(f"no alerts task in {run_manifest} for {sorted(unknown)}")
frozen = frozen_source(source)
scratch = Path(work) if work is not None else Path(tempfile.mkdtemp(prefix="ousast-engine-alerts-"))
results: list[dict[str, Any]] = []
for case in cases:
if only and case.case_id not in only:
continue
results.append(
produce(case, run_name, frozen, scratch, cache=cache, deadline=deadline, image=image, runner=runner, results_root=results_root)
)
if work is None:
shutil.rmtree(scratch, ignore_errors=True)
return results
__all__ = ["EngineAlertsError", "EngineCase", "alerts_engine", "engine_cases", "engine_rows", "parse_site", "produce", "read_record"]
src/openultrasast/plane/generate.py
+2 -1
View File
@@ -314,6 +314,7 @@ def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str,
inputs["agreed"] = f"{case_id}-final/agreed.json"
if alerts:
inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl"
inputs["alerts_summary"] = f"{case_id}-alerts/summary.json" # the coverage: whether a zero could mean clean
entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()}
return task, entry
@@ -332,7 +333,7 @@ def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict
"OUSAST_FIXED_PIN": str(case["fixed"]),
} # fmt: skip
task = _task(templates["alerts"], f"alerts-{case['id']}", env, bindings)
entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl"], "budget": _free()}
entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl", "summary.json"], "budget": _free()}
entry["serialize"] = "engine"
return fixed, task, entry
src/openultrasast/plane/memory.py
+2 -2
View File
@@ -1,7 +1,7 @@
"""The plane's memory store (harnessx-removal Requirement 6.1/6.4, design section 4).
One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows
of five kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``proposal_outcome``) that the ``remember`` task
of six kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``proposal_outcome``) that ``remember``
(:mod:`.tasks.remember`) derives from a run's delivered artifacts. Layout, identical for both backends::
index.jsonl one row per ingested (run, task): sha256 of its rows, row count
@@ -49,7 +49,7 @@ MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key"
POPULATION_ANNOTATION = "openultrasast.io/population"
SPLIT_ANNOTATION = "openultrasast.io/split"
SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"}
KINDS = ("facts", "verdict", "unit_cost", "alert", "proposal_outcome")
KINDS = ("facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome")
ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image")
TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split")
MIN_FREE_BYTES = 1 << 30
src/openultrasast/plane/tasks/alerts.py
+88 -16
View File
@@ -15,6 +15,15 @@ of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fix
none). A fixed-pin alert inside the fix is a false alert on code the fix wrote: rule M1 of ``improve/memory.py``
counts it.
**Coverage** (an honest zero): quick mode has rules only for the languages its ruleset declares
(:func:`quick_languages`, derived from the bundled ruleset directory, never listed by hand). ``summary.json`` records
per language of the files the scan read whether quick mode covers it -- ``coverage: {language: {"coverage": "quick" |
"none", "files": {role: n}}}`` -- and ``uncovered``, the languages with ``none``. Zero alerts on an uncovered
language is not a clean result: nothing could have fired. ``remember`` turns the coverage into ``coverage`` rows,
and M1/M2 skip a repository and pin whose coverage for a rule's language is ``none``. The deterministic engine covers
more languages (:func:`engine_languages`, PHP among them); ``ousast plane alerts-engine`` (:mod:`..engine_alerts`)
produces those alerts on the host, where the engine image runs, and marks this task done.
Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_WORKSPACE_DIR`` (vulnerable checkout), ``OUSAST_FIXED_DIR`` (fixed checkout),
``OUSAST_INPUT_CASE`` (``case.json``), ``OUSAST_VULNERABLE_PIN``, ``OUSAST_FIXED_PIN``. A pin whose checkout yields no
source file fails the task: an unread tree and a silent one are indistinguishable afterwards. Exit 0 done, 2 failed.
@@ -26,7 +35,7 @@ import json
import os
import sys
import traceback
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from pathlib import Path
from typing import Any
@@ -34,14 +43,55 @@ from ...findings import quick_scan_findings
from ...mapping import analyze_entry_points, attach_reachability_hints
from ...preprocess import detect_language, preprocess_repository
from ...rank import rank_targets
from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset
from .repo_facts import DECLARATION, GLOBAL, enclosing
ROLES = ("vulnerable", "fixed")
DIRS = ("OUSAST_WORKSPACE_DIR", "OUSAST_FIXED_DIR")
PINS = ("OUSAST_VULNERABLE_PIN", "OUSAST_FIXED_PIN")
ANY_LANGUAGE = "*" # a rule that names no language runs on every file
Ranges = Mapping[str, Sequence[Sequence[int]]]
def quick_languages(directory: Path = DEFAULT_RULESET_DIR) -> frozenset[str]:
"""The languages quick mode has a rule for: the ``languages`` of every rule the ruleset directory holds that is
not disabled (``*`` when a rule names none: it runs on every file)."""
covered: set[str] = set()
for rule in load_ruleset(directory):
if rule.status != "disabled":
covered.update(rule.languages or (ANY_LANGUAGE,))
return frozenset(covered)
def engine_languages(directory: Path = DEFAULT_RULESET_DIR) -> frozenset[str]:
"""The languages the deterministic engine has a semantic model for: one ``semantic/<language>.toml`` each."""
return frozenset(path.stem for path in (directory / "semantic").glob("*.toml"))
def covers(covered: Iterable[str], language: str) -> bool:
languages = set(covered)
return ANY_LANGUAGE in languages or language in languages
def coverage(files: Mapping[str, Mapping[str, int]], quick: Iterable[str], engine: Iterable[str] = ()) -> dict[str, dict[str, Any]]:
"""Per language of the files read (``{role: {language: n}}``): ``quick`` when quick mode has a rule for it,
``engine`` when the engine's alerts were produced for it, else ``none``, with the file count per role."""
quick, engine = set(quick), set(engine)
result: dict[str, dict[str, Any]] = {}
for role in sorted(files):
for language, number in files[role].items():
if language == "unknown":
continue
kind = "quick" if covers(quick, language) else "engine" if language in engine else "none"
entry = result.setdefault(language, {"coverage": kind, "files": {}})
entry["files"][role] = int(number)
return dict(sorted(result.items()))
def uncovered(cov: Mapping[str, Mapping[str, Any]]) -> list[str]:
return sorted(language for language, entry in cov.items() if entry.get("coverage") == "none")
def _lines(path: Path) -> list[str]:
try:
return path.read_text(encoding="utf-8", errors="replace").splitlines()
@@ -55,24 +105,35 @@ def in_range(ranges: Ranges | None, path: str, line: int | None) -> bool | None:
return line is not None and any(int(lo) <= line <= int(hi) for lo, hi in ranges.get(path, []))
def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, int]]:
"""(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line."""
def function_at(root: Path, path: str, line: int | None, fallback: str | None, cache: dict[str, list[str]]) -> str:
"""The enclosing declaration of ``path:line`` by the ``repo-facts`` patterns, else ``fallback`` (``<global>``)."""
lines = cache.setdefault(path, _lines(root / path))
language = detect_language(root / path)
if language in DECLARATION and line and 0 < line <= len(lines):
return enclosing(lines, line - 1, language)
return fallback or GLOBAL
def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""(rows, what was read) of one pin: the quick-mode findings as alert rows, one per rule, file, line; what was read
counts files and bytes, and files per language."""
_, targets = preprocess_repository(root)
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
findings = quick_scan_findings(root, targets, rank_targets(targets))
read = {"files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file())}
languages: dict[str, int] = {}
for target in targets:
languages[target.language] = languages.get(target.language, 0) + 1
read = {
"files": len(targets), "bytes": sum((root / t.path).stat().st_size for t in targets if (root / t.path).is_file()),
"languages": dict(sorted(languages.items())),
} # fmt: skip
cache: dict[str, list[str]] = {}
rows: dict[tuple[str, str, int | None], dict[str, Any]] = {}
for finding in findings:
rule_id = finding.finding_id.split(":", 1)[0]
lines = cache.setdefault(finding.path, _lines(root / finding.path))
language = detect_language(root / finding.path)
if language in DECLARATION and finding.line and 0 < finding.line <= len(lines):
function = enclosing(lines, finding.line - 1, language)
else:
function = finding.function_name or GLOBAL
row = {
"rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line, "function": function,
"rule_id": rule_id, "rule_status": finding.status, "path": finding.path, "line": finding.line,
"function": function_at(root, finding.path, finding.line, finding.function_name, cache),
"pin_role": role, "pin": pin, "in_fix_range": in_range(ranges, finding.path, finding.line),
} # fmt: skip
rows.setdefault((rule_id, finding.path, finding.line), row)
@@ -80,10 +141,14 @@ def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[d
return ordered, read
def _summary(output_dir: Path, summary: Mapping[str, Any]) -> None:
def write_summary(output_dir: Path, summary: Mapping[str, Any]) -> None:
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def write_rows(output_dir: Path, rows: Iterable[Mapping[str, Any]]) -> None:
(output_dir / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
def main(environ: Mapping[str, str] | None = None) -> int:
env = environ if environ is not None else os.environ
output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None
@@ -95,6 +160,7 @@ def main(environ: Mapping[str, str] | None = None) -> int:
case = json.loads(Path(env["OUSAST_INPUT_CASE"]).read_text(encoding="utf-8")) if env.get("OUSAST_INPUT_CASE") else {}
ranges = {"vulnerable": case.get("ranges"), "fixed": case.get("fixed_ranges")}
rows: list[dict[str, Any]] = []
files: dict[str, dict[str, int]] = {}
for role, directory, pin in zip(ROLES, DIRS, PINS, strict=True):
root = Path(env.get(directory) or "")
if not env.get(directory) or not root.is_dir():
@@ -102,21 +168,27 @@ def main(environ: Mapping[str, str] | None = None) -> int:
found, read = scan(root, role, str(env.get(pin) or ""), ranges[role])
if read["files"] == 0:
raise ValueError(f"the {role} checkout {root} has no source file the scan reads: an unread tree is not a clean one")
files[role] = read.pop("languages")
summary.setdefault("read", {})[role] = read
summary.setdefault("alerts", {})[role] = len(found)
summary.setdefault("pins", {})[role] = str(env.get(pin) or "")
rows.extend(found)
summary["units_done"] += 1
(output / "alerts.jsonl").write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
quick = quick_languages()
summary["quick_languages"] = sorted(quick)
summary["coverage"] = coverage(files, quick)
summary["uncovered"] = uncovered(summary["coverage"])
write_rows(output, rows)
summary["status"] = "done"
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
summary["reason"] = traceback.format_exc()[-2000:]
if output is not None:
_summary(output, summary)
print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read")}), file=sys.stderr)
write_summary(output, summary)
print(json.dumps({k: summary.get(k) for k in ("status", "alerts", "read", "uncovered")}), file=sys.stderr)
return 0 if summary["status"] == "done" else 2
__all__ = ["in_range", "main", "scan"]
__all__ = ["coverage", "covers", "engine_languages", "function_at", "in_range", "main", "quick_languages", "scan", "uncovered"]
if __name__ == "__main__":
src/openultrasast/plane/tasks/loop.py
+5
View File
@@ -119,6 +119,10 @@ def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
tally_a = alerts.setdefault(str(a.get("rule_id")), {"vulnerable": 0, "fixed": 0, "fixed_in_fix_range": 0})
tally_a[role] += 1
tally_a["fixed_in_fix_range"] += role == "fixed" and a.get("in_fix_range") is True
coverage: dict[str, dict[str, int]] = {} # language -> pins per coverage kind: a `none` pin's zero alerts mean nothing
for c in (r for r in rows if r["kind"] == "coverage"):
tally_c = coverage.setdefault(str(c.get("language")), {"quick": 0, "engine": 0, "none": 0})
tally_c[str(c.get("coverage"))] = tally_c.get(str(c.get("coverage")), 0) + 1
return {
"cases": len({(v["repo"], v["pin"]) for v in verdicts}),
"candidates": len(verdicts),
@@ -130,6 +134,7 @@ def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
"usd_per_candidate": None if usd is None or not verdicts else round(usd / len(verdicts), 6),
"dispute_rate": {f: {"candidates": n, "disputed": d, "rate": round(d / n, 4)} for f, (n, d) in sorted(families.items())},
"alerts": dict(sorted(alerts.items())),
"coverage": dict(sorted(coverage.items())),
}
src/openultrasast/plane/tasks/remember.py
+36 -7
View File
@@ -2,8 +2,9 @@
No model, budget ``{usd: 0, calls: 0}``, one per case. It reads the case's `facts.json`, the `units.jsonl` of the
verify passes a, b and c, the final `agreed.json` (its per-candidate rows and `disputed` list) and, when present,
`alerts.jsonl`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`, `unit_cost` and `alert` (the
fifth kind, `proposal_outcome`, comes from the loop) -- and passes `facts.json` through. Task code interprets the
`alerts.jsonl` with the alerts' `summary.json`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`,
`unit_cost`, `alert` and `coverage` (per language and pin: whether any rule could have fired there; the kind
`proposal_outcome` comes from the loop) -- and passes `facts.json` through. Task code interprets the
artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Req 3.4).
Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`,
@@ -12,7 +13,8 @@ Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`,
Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS` (and `OUSAST_INPUT_FACTS_SUMMARY`,
whose reused or recorded `image` the facts row carries; without one named, the row is omitted),
`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes'
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS` (and `OUSAST_INPUT_ALERTS_SUMMARY`,
the coverage); the repository and pin from the bound
Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from
`OUSAST_POPULATION`/`OUSAST_SPLIT` (the generator copies the Run's `openultrasast.io/population` and
`openultrasast.io/split` annotations). Exit 0 done, 2 failed.
@@ -74,6 +76,26 @@ def _final(row: Mapping[str, Any], disputed: set[str]) -> str:
return "disputed" if str(row["candidate"]) in disputed else "rejected"
def coverage_rows(ctx: Context, alerts_summary: Mapping[str, Any] | None) -> list[dict[str, Any]]:
"""One ``coverage`` row per language and pin of the alerts' ``summary.json``: ``quick``, ``engine`` or ``none``
(no rule could fire, so no alert there is evidence of anything), with the files read in that language. A summary
from before coverage was recorded yields none: unknown stays unknown."""
if not alerts_summary or not isinstance(alerts_summary.get("coverage"), Mapping):
return []
recorded = alerts_summary.get("pins")
pins: Mapping[str, Any] = recorded if isinstance(recorded, Mapping) else {}
rows: list[dict[str, Any]] = []
for language, entry in sorted(alerts_summary["coverage"].items()):
for role, files in sorted((entry.get("files") or {}).items()):
fields: dict[str, Any] = {"language": language, "coverage": entry.get("coverage"), "files": files, "pin_role": role}
if pins.get(role):
fields["pin"] = str(pins[role])
if alerts_summary.get("source"):
fields["source"] = str(alerts_summary["source"])
rows.append(ctx.row("coverage", f"{language}:{role}", **fields))
return rows
def rows_for(
ctx: Context,
*,
@@ -82,8 +104,9 @@ def rows_for(
models: Mapping[str, str | None],
agreed: Mapping[str, Any] | None,
alerts: Sequence[Mapping[str, Any]] = (),
alerts_summary: Mapping[str, Any] | None = None,
) -> list[dict[str, Any]]:
"""The memory rows of one case: facts, verdicts, unit costs, alerts."""
"""The memory rows of one case: facts, verdicts, unit costs, alerts, and the alerts' coverage per language and pin."""
rows: list[dict[str, Any]] = []
if facts is not None:
loaded = json.loads(facts)
@@ -115,8 +138,11 @@ def rows_for(
fields["in_fix_range"] = alert["in_fix_range"]
if alert.get("pin"): # a fixed-pin alert is a row of the pin it was raised on, not of the case's vulnerable pin
fields["pin"] = str(alert["pin"])
if alert.get("source"): # `engine` when `ousast plane alerts-engine` produced it on the host
fields["source"] = str(alert["source"])
subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "")
rows.append(ctx.row("alert", subject, **fields))
rows.extend(coverage_rows(ctx, alerts_summary))
return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows]
@@ -185,7 +211,9 @@ def case_rows(base: Path, case: str, *, population: str, split: str) -> tuple[li
passes = {p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES}
models = {p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES}
alerts = _jsonl(base / f"{case}-alerts" / "alerts.jsonl")
return rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts), facts
alerts_summary = _json(base / f"{case}-alerts" / "summary.json")
rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary)
return rows, facts
def remember_run(run_dir: Path, store: MemoryStore, *, population: str = UNKNOWN, split: str = UNKNOWN) -> list[IngestResult]:
@@ -246,7 +274,8 @@ def main(environ: Mapping[str, str] | None = None) -> int:
passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES}
models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES}
agreed = _json(_path(env, "OUSAST_INPUT_AGREED"))
rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")))
alerts, alerts_summary = _jsonl(_path(env, "OUSAST_INPUT_ALERTS")), _json(_path(env, "OUSAST_INPUT_ALERTS_SUMMARY"))
rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary)
facts_image = _facts_image(env, ctx.image)
if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on
facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split)
@@ -265,7 +294,7 @@ def main(environ: Mapping[str, str] | None = None) -> int:
return 0 if summary["status"] == "done" else 2
__all__ = ["Context", "candidates_digest", "case_rows", "context_from", "main", "remember_run", "rows_for"]
__all__ = ["Context", "candidates_digest", "case_rows", "context_from", "coverage_rows", "main", "remember_run", "rows_for"]
if __name__ == "__main__":
tests/fixtures/plane-engine/app/index.php
Vendored
+5
View File
@@ -0,0 +1,5 @@
<?php
$id = $_GET["id"];
$r = mysqli_query($conn, "SELECT * FROM users WHERE id = " . $id);
echo $_GET["name"];
system("ping " . $_GET["host"]);
tests/fixtures/plane-engine/probe.json
Vendored
+29
View File
@@ -0,0 +1,29 @@
{
"root": "/case",
"families": [
"injection"
],
"seconds": 58.6,
"questions": 1,
"completed": 1,
"completed_regions": [
"app/index.php:"
],
"degradations": [],
"findings": [
{
"site": "app/index.php:3:<global>",
"family": "injection",
"rung": "model_entailed",
"witness": "$_GET[\"id\"] -> mysqli_query($conn, \"SELECT * FROM users WHERE id = \" . $id) in app/index.php (line 3, 4 steps)",
"reached_from": 1
},
{
"site": "app/index.php:5:<global>",
"family": "injection",
"rung": "model_entailed",
"witness": "$_GET[\"host\"] -> system(\"ping \" . $_GET[\"host\"]) in app/index.php (line 5, 2 steps)",
"reached_from": 1
}
]
}
tests/fixtures/plane-engine/probe.stdout.txt
Vendored
+2
View File
@@ -0,0 +1,2 @@
read 1 files, 145 bytes; 1 regions offer ('injection',)
{"root": "/case", "families": ["injection"], "seconds": 58.6, "questions": 1, "completed": 1, "degradations": [], "findings": 2}
tests/test_improve_memory.py
+28
View File
@@ -149,6 +149,34 @@ def test_m2_stays_silent_on_a_fixed_pin_alert_or_rejected_candidates() -> None:
assert propose([*site_hits(), *rejected], rules) == []
# ---- coverage: no rule could fire, so no evidence either way -------------------------------------------------------
def covered(repo: str, pin: str, kind: str, language: str = "python", role: str = "vulnerable") -> dict[str, Any]:
return _row("coverage", repo, pin, f"{language}:{role}:{kind}", language=language, coverage=kind, files=3, pin_role=role)
def test_m1_skips_a_repository_and_pin_whose_coverage_for_the_rules_language_is_none() -> None:
blind = [*false_alerts(), covered("r2", P["a"], "none")]
assert propose(blind, [_rule("noisy")]) == [], "r2's alert could not be evidence: one repository is left"
wider = [*false_alerts(repos=("r1", "r1", "r2", "r3")), covered("r3", P["a"], "none")]
[proposal] = propose(wider, [_rule("noisy")])
assert proposal.provenance["counts"] == {"false_alerts": 3, "repos": 2, "agreed_hits": 0}
assert proposal.provenance["excluded"]["coverage_none"] == {f"github.com/o/r3@{P['a']}": 1}
def test_coverage_none_of_another_language_or_beside_engine_coverage_is_no_skip() -> None:
other = [*false_alerts(), covered("r2", P["a"], "none", language="php")]
assert len(propose(other, [_rule("noisy")])) == 1, "the rule's language (python) was covered there"
engine = [*false_alerts(), covered("r2", P["a"], "none"), covered("r2", P["a"], "engine")]
assert len(propose(engine, [_rule("noisy")])) == 1, "a later run covered the pin: its alerts are evidence"
def test_m2_skips_a_blind_repository_and_pin() -> None:
rules = [_rule("quiet", status="shadow"), _rule("other")]
assert propose([*site_hits(), covered("r2", P["b"], "none")], rules) == []
# ---- M3: advisory only ---------------------------------------------------------------------------------------------
tests/test_plane_engine_alerts.py
+160
View File
@@ -0,0 +1,160 @@
"""`ousast plane alerts-engine` (harnessx-removal Req 6.3): PHP alerts from the engine image, produced on the host.
No container runs here: the docker runner is injected. The engine's output is the recorded probe
(``fixtures/plane-engine/probe.json``: 1 file read, 1 question asked and completed, 2 injection findings)."""
from __future__ import annotations
import json
import shutil
import subprocess
from collections.abc import Sequence
from pathlib import Path
from typing import Any
import pytest
from test_plane_loop import LOOP, _templates, _write
from openultrasast import cli
from openultrasast.plane import engine_alerts
from openultrasast.plane.engine_alerts import EngineAlertsError, alerts_engine, engine_rows, parse_site, read_record
from openultrasast.plane.generate import CaseInputs, render
FIXTURE = Path(__file__).parent / "fixtures" / "plane-engine"
CASE = "php-probe"
RECORD = json.loads((FIXTURE / "probe.json").read_text())
LOG = (FIXTURE / "probe.stdout.txt").read_text()
def _git(repo: Path, *args: str) -> str:
return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=True).stdout.strip()
@pytest.fixture
def world(tmp_path: Path) -> dict[str, Any]:
"""A case cache with a vulnerable and a fixed commit, a Run with the case's `alerts` task, a frozen source."""
repo = tmp_path / "cache" / CASE
(repo / "app").mkdir(parents=True)
_git(repo, "init", "-q")
shutil.copy(FIXTURE / "app" / "index.php", repo / "app" / "index.php")
_git(repo, "add", ".")
_git(repo, "-c", "user.name=t", "-c", "[email protected]", "commit", "-qm", "vulnerable")
vulnerable = _git(repo, "rev-parse", "HEAD")
text = (repo / "app" / "index.php").read_text().replace('" . $id', '" . intval($id)')
(repo / "app" / "index.php").write_text(text)
_git(repo, "-c", "user.name=t", "-c", "[email protected]", "commit", "-qam", "fixed")
fixed = _git(repo, "rev-parse", "HEAD")
record = {"id": CASE, "repo": "https://example.com/reserved/probe", "family": "injection", "vulnerable": vulnerable, "fixed": fixed}
candidate = ("app/index.php", "<global>", 3)
case = CaseInputs(
record, (candidate,), (candidate,), {}, 0.0, ranges={"app/index.php": [(3, 3)]}, fixed_ranges={"app/index.php": [(3, 3)]}
)
files = render([case], _templates(), "validation-46", "t", population="p", split="s", loop=LOOP)
source = tmp_path / "frozen"
(source / "src" / "openultrasast").mkdir(parents=True)
(source / "benchmarks" / "push").mkdir(parents=True)
(source / "benchmarks" / "push" / "finding_dump.py").write_text("")
run = _write(tmp_path / "plane", files)
return {"run": run, "source": source, "cache": tmp_path / "cache", "pins": (vulnerable, fixed), "results": tmp_path / "results"}
class FakeDocker:
"""Answers `docker image inspect` and writes ``record`` as the engine's result for every `docker run`."""
def __init__(self, record: dict[str, Any] | None = RECORD, log: str = LOG) -> None:
self.record, self.log, self.runs = record, log, [] # type: ignore[var-annotated]
def __call__(self, command: Sequence[str], timeout: float) -> subprocess.CompletedProcess[str]:
command = list(command)
if command[:3] == ["docker", "image", "inspect"]:
return subprocess.CompletedProcess(command, 0, "sha256:feed\n", "")
mounts = [command[i + 1] for i, part in enumerate(command) if part == "-v"]
checkout = Path(next(m for m in mounts if m.endswith(":/case:ro")).split(":")[0])
assert (checkout / "app" / "index.php").is_file(), "the pin was exported before the container ran"
self.runs.append({"command": command, "timeout": timeout})
if self.record is not None:
out = Path(next(m for m in mounts if m.endswith(":/out")).split(":")[0])
(out / command[command.index("--out") + 1].removeprefix("/out/")).write_text(json.dumps(self.record))
return subprocess.CompletedProcess(command, 0, self.log, "")
def _produce(world: dict[str, Any], docker: FakeDocker, tmp_path: Path) -> list[dict[str, Any]]:
return alerts_engine(
world["run"], source=world["source"], work=tmp_path / "work", cache=world["cache"], deadline=120, runner=docker,
results_root=world["results"],
) # fmt: skip
def test_recorded_engine_findings_become_alert_rows() -> None:
assert parse_site("app/index.php:3:<global>") == ("app/index.php", 3, "<global>")
assert parse_site("lib/A.php:76:Foo::bar") == ("lib/A.php", 76, "Foo::bar")
record = read_record(FIXTURE / "probe.json", LOG, 0)
assert (record["files"], record["bytes"], record["questions"], record["completed"]) == (1, 145, 1, 1)
rows = engine_rows(record, FIXTURE, "vulnerable", "a" * 40, {"app/index.php": [[3, 3]]})
assert rows == [
{"rule_id": "engine:injection", "rule_status": "enabled", "path": "app/index.php", "line": line, "function": "<global>",
"pin_role": "vulnerable", "pin": "a" * 40, "in_fix_range": line == 3, "source": "engine"}
for line in (3, 5)
] # fmt: skip
def test_an_engine_that_read_nothing_or_asked_nothing_fails(tmp_path: Path) -> None:
out = tmp_path / "x.json"
with pytest.raises(EngineAlertsError, match="wrote no result"):
read_record(out, "boom", 125)
out.write_text(json.dumps({**RECORD, "questions": 0}))
with pytest.raises(EngineAlertsError, match="no question was asked"):
read_record(out, LOG, 0)
out.write_text(json.dumps(RECORD))
with pytest.raises(EngineAlertsError, match="did not report reading"):
read_record(out, "", 0)
def test_alerts_engine_writes_the_alerts_task_outputs_and_marks_it_done(world: dict[str, Any], tmp_path: Path) -> None:
docker = FakeDocker()
[result] = _produce(world, docker, tmp_path)
assert result["status"] == "done" and len(docker.runs) == 2, "one container per pin, one after the other"
for run in docker.runs:
command = run["command"]
assert command[command.index("--memory") + 1] == "3g" and command[command.index("--network") + 1] == "none"
assert command[command.index("--families") + 1] == "injection" and run["timeout"] == 120 + engine_alerts.GRACE
out = world["results"] / "validation-46" / f"{CASE}-alerts"
rows = [json.loads(line) for line in (out / "alerts.jsonl").read_text().splitlines()]
vulnerable, fixed = world["pins"]
got = [(r["pin_role"], r["pin"], r["line"], r["in_fix_range"], r["source"]) for r in rows]
assert got == [("vulnerable", vulnerable, 3, True, "engine"), ("vulnerable", vulnerable, 5, False, "engine"),
("fixed", fixed, 3, True, "engine"), ("fixed", fixed, 5, False, "engine")] # fmt: skip
summary = json.loads((out / "summary.json").read_text())
assert (summary["status"], summary["units_done"], summary["units_total"], summary["source"]) == ("done", 2, 2, "engine")
assert summary["engine"]["vulnerable"] == {
"files": 1, "bytes": 145, "questions": 1, "completed": 1, "seconds": 58.6, "degradations": [], "exit": 0, "findings": 2
} # fmt: skip
assert summary["coverage"] == {"php": {"coverage": "engine", "files": {"fixed": 1, "vulnerable": 1}}} and summary["uncovered"] == []
assert summary["in_fix_range"] == {"vulnerable": 1, "fixed": 1} and summary["image"] == "sha256:feed"
assert isinstance(summary["seconds"], float)
state = json.loads((world["results"] / "validation-46" / "state.json").read_text())["tasks"][f"{CASE}-alerts"]
assert state["status"] == "done" and state["reused"]["source"] == "engine", "the reconciler skips it like reused facts"
assert not list((tmp_path / "work").glob(f"{CASE}--*/"))
def test_a_failed_engine_leaves_the_run_untouched(world: dict[str, Any], tmp_path: Path) -> None:
with pytest.raises(EngineAlertsError, match="no question was asked"):
_produce(world, FakeDocker({**RECORD, "questions": 0, "findings": []}), tmp_path)
with pytest.raises(EngineAlertsError, match="wrote no result"):
_produce(world, FakeDocker(None), tmp_path)
assert not (world["results"] / "validation-46").exists(), "no alerts written, nothing marked done"
def test_the_source_must_be_a_frozen_export(world: dict[str, Any], tmp_path: Path) -> None:
(world["source"] / ".git").mkdir()
with pytest.raises(EngineAlertsError, match="export it first"):
_produce(world, FakeDocker(), tmp_path)
def test_the_cli_reports_and_fails_loudly(world: dict[str, Any], tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None: # fmt: skip
monkeypatch.setattr(engine_alerts, "run_command", FakeDocker({**RECORD, "questions": 0}))
argv = ["plane", "alerts-engine", str(world["run"]), "--source", str(world["source"]), "--repos", str(world["cache"])]
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "r"))
assert cli.main(argv) == 2
assert "no question was asked" in capsys.readouterr().err
tests/test_plane_loop.py
+34
View File
@@ -97,6 +97,39 @@ def test_alerts_attribute_functions_and_mark_the_fix_on_both_pins(tmp_path: Path
assert "an unread tree is not a clean one" in json.loads((out / "summary.json").read_text())["reason"]
def test_quick_coverage_is_derived_from_the_ruleset_directory(tmp_path: Path) -> None:
shipped = alerts.quick_languages()
assert {"python", "javascript", "typescript", "java", "c", "groovy"} <= shipped and "php" not in shipped
assert "php" in alerts.engine_languages(), "the engine's semantic models cover PHP"
rules = tmp_path / "ruleset" / "php"
rules.mkdir(parents=True)
rule = 'rule_id = "php-x"\ntitle = "x"\nlanguages = ["php"]\ncwe = "CWE-89"\ntags = []\npattern = "x"\n'
(rules / "rules.toml").write_text(f"[[rule]]\n{rule}")
assert alerts.quick_languages(tmp_path / "ruleset") == {"php"}, "a ruleset added for a language covers it"
(rules / "rules.toml").write_text(f'[[rule]]\n{rule}status = "disabled"\n')
assert alerts.quick_languages(tmp_path / "ruleset") == set(), "a disabled rule fires nowhere"
def test_alerts_on_an_uncovered_language_record_coverage_none_not_a_clean_zero(tmp_path: Path) -> None:
for side in ("v", "f"):
(tmp_path / side).mkdir()
(tmp_path / side / "index.php").write_text('<?php\n$id = $_GET["id"];\nmysqli_query($c, "SELECT " . $id);\n')
(tmp_path / side / "app.js").write_text("module.exports = 1;\n")
out = tmp_path / "out"
env = {
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_WORKSPACE_DIR": str(tmp_path / "v"), "OUSAST_FIXED_DIR": str(tmp_path / "f"),
"OUSAST_VULNERABLE_PIN": VULNERABLE, "OUSAST_FIXED_PIN": FIXED,
} # fmt: skip
assert alerts.main(env) == 0
summary = json.loads((out / "summary.json").read_text())
assert (out / "alerts.jsonl").read_text() == "" and summary["alerts"] == {"vulnerable": 0, "fixed": 0}
assert summary["coverage"] == {
"javascript": {"coverage": "quick", "files": {"fixed": 1, "vulnerable": 1}},
"php": {"coverage": "none", "files": {"fixed": 1, "vulnerable": 1}},
}
assert summary["uncovered"] == ["php"] and summary["pins"] == {"vulnerable": VULNERABLE, "fixed": FIXED}
def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None:
repo = tmp_path / "r"
repo.mkdir()
@@ -129,6 +162,7 @@ def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path)
names = [e.name for e in run.tasks]
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS]
assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl"
assert run.task(f"{CASE}-remember").inputs["alerts_summary"] == f"{CASE}-alerts/summary.json", "the coverage travels too"
measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"}
assert run.task("loop-measure").inputs == measure_inputs
assert run.task("loop-improve").producers == ("loop-propose",)
tests/test_plane_remember.py
+32
View File
@@ -99,6 +99,38 @@ def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
COVERAGE_SUMMARY = {
"status": "done", "pins": {"vulnerable": "0" * 40, "fixed": "f" * 40},
"coverage": {
"javascript": {"coverage": "quick", "files": {"vulnerable": 2}},
"php": {"coverage": "none", "files": {"vulnerable": 9, "fixed": 9}},
},
} # fmt: skip
def test_the_alerts_coverage_becomes_one_row_per_language_and_pin() -> None:
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts=[], alerts_summary=COVERAGE_SUMMARY)
got = sorted((r["language"], r["pin_role"], r["pin"], r["coverage"], r["files"]) for r in rows if r["kind"] == "coverage")
assert got == [
("javascript", "vulnerable", "0" * 40, "quick", 2), ("php", "fixed", "f" * 40, "none", 9),
("php", "vulnerable", "0" * 40, "none", 9),
] # fmt: skip
assert rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts_summary={"status": "done"}) == [], "unknown stays unknown"
def test_remember_run_carries_the_alerts_coverage_into_the_store(run_dir: Path, tmp_path: Path) -> None:
(run_dir / f"{CASE}-alerts").mkdir()
(run_dir / f"{CASE}-alerts" / "alerts.jsonl").write_text("") # zero alerts: meaningful only through the coverage
summary = {**COVERAGE_SUMMARY, "pins": {"vulnerable": PIN, "fixed": "f" * 40}}
(run_dir / f"{CASE}-alerts" / "summary.json").write_text(json.dumps(summary))
store = FileStore(tmp_path / "memory")
[result] = remember_run(run_dir, store)
assert result.kinds["coverage"] == 3
php = sorted((r["pin"], r["coverage"]) for r in _by_kind(store)["coverage"] if r["language"] == "php")
assert php == [(PIN, "none"), ("f" * 40, "none")]
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
task = next(d for d in docs if d["kind"] == "Task")