mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(plane): honest coverage in alerts; PHP alerts from the engine on the host
Quick mode has rules only for c/cpp, groovy, java, javascript/typescript and python, so every PHP case's "0 alerts" read as clean. The `alerts` task now records per language of the files read whether quick mode covers it (derived from the bundled ruleset, not a list): coverage quick | none, with file counts per pin, and `uncovered`. `remember` turns that into `coverage` memory rows (a new kind; the Run passes the alerts summary.json as input), loop-measure reports coverage per language, and improve/memory.py's M1/M2 skip a repository+pin whose coverage for the rule's language is only `none` (counted under `coverage_none` in the proposal's excluded). `ousast plane alerts-engine <Run.yaml> --source <frozen export>` (plane/engine_alerts.py) does the alerts task on the host for cases whose files include a language the engine covers but quick mode does not (PHP): exports both pins from the case cache, runs the quick scan and the openultrasast:dev engine container (finding_dump.py, --network none, --memory 3g, one at a time, per-pin deadline), writes engine findings in the alerts.jsonl row shape (rule_id engine:<family>, source engine, in_fix_range from case.json) with a summary (files read, questions, completed, seconds, degradations per pin), and marks the task done via reconciler.mark_done. No result, no files read, or questions == 0 fails loudly and leaves the run untouched. Refuses a git checkout as source. Runner image rebuilt and re-pinned (sha256:ebb40a9a...); validation-46 regenerated (digest only). Live, recorded in benchmarks/measurements/2026-09-30-php-engine-alerts/: froxlor 366 files, 130/130 questions per pin, 6 alerts per pin, 0 in the fix range; wp-gopay 19 files, 101/101 questions, 0 alerts. The loop: 32 run rows (coverage 8, php=engine), 0 proposals, gate no_proposals, $0. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f4bd984fda
commit
45d02ecf20
@@ -99,6 +99,38 @@ def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
|
||||
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
|
||||
|
||||
|
||||
COVERAGE_SUMMARY = {
|
||||
"status": "done", "pins": {"vulnerable": "0" * 40, "fixed": "f" * 40},
|
||||
"coverage": {
|
||||
"javascript": {"coverage": "quick", "files": {"vulnerable": 2}},
|
||||
"php": {"coverage": "none", "files": {"vulnerable": 9, "fixed": 9}},
|
||||
},
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def test_the_alerts_coverage_becomes_one_row_per_language_and_pin() -> None:
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts=[], alerts_summary=COVERAGE_SUMMARY)
|
||||
got = sorted((r["language"], r["pin_role"], r["pin"], r["coverage"], r["files"]) for r in rows if r["kind"] == "coverage")
|
||||
assert got == [
|
||||
("javascript", "vulnerable", "0" * 40, "quick", 2), ("php", "fixed", "f" * 40, "none", 9),
|
||||
("php", "vulnerable", "0" * 40, "none", 9),
|
||||
] # fmt: skip
|
||||
assert rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts_summary={"status": "done"}) == [], "unknown stays unknown"
|
||||
|
||||
|
||||
def test_remember_run_carries_the_alerts_coverage_into_the_store(run_dir: Path, tmp_path: Path) -> None:
|
||||
(run_dir / f"{CASE}-alerts").mkdir()
|
||||
(run_dir / f"{CASE}-alerts" / "alerts.jsonl").write_text("") # zero alerts: meaningful only through the coverage
|
||||
summary = {**COVERAGE_SUMMARY, "pins": {"vulnerable": PIN, "fixed": "f" * 40}}
|
||||
(run_dir / f"{CASE}-alerts" / "summary.json").write_text(json.dumps(summary))
|
||||
store = FileStore(tmp_path / "memory")
|
||||
[result] = remember_run(run_dir, store)
|
||||
assert result.kinds["coverage"] == 3
|
||||
php = sorted((r["pin"], r["coverage"]) for r in _by_kind(store)["coverage"] if r["language"] == "php")
|
||||
assert php == [(PIN, "none"), ("f" * 40, "none")]
|
||||
|
||||
|
||||
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
|
||||
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
|
||||
task = next(d for d in docs if d["kind"] == "Task")
|
||||
|
||||
Reference in New Issue
Block a user