feat(plane): honest coverage in alerts; PHP alerts from the engine on the host

Quick mode has rules only for c/cpp, groovy, java, javascript/typescript and
python, so every PHP case's "0 alerts" read as clean. The `alerts` task now
records per language of the files read whether quick mode covers it
(derived from the bundled ruleset, not a list): coverage quick | none, with
file counts per pin, and `uncovered`. `remember` turns that into `coverage`
memory rows (a new kind; the Run passes the alerts summary.json as input),
loop-measure reports coverage per language, and improve/memory.py's M1/M2
skip a repository+pin whose coverage for the rule's language is only
`none` (counted under `coverage_none` in the proposal's excluded).

`ousast plane alerts-engine <Run.yaml> --source <frozen export>`
(plane/engine_alerts.py) does the alerts task on the host for cases whose
files include a language the engine covers but quick mode does not (PHP):
exports both pins from the case cache, runs the quick scan and the
openultrasast:dev engine container (finding_dump.py, --network none,
--memory 3g, one at a time, per-pin deadline), writes engine findings in
the alerts.jsonl row shape (rule_id engine:<family>, source engine,
in_fix_range from case.json) with a summary (files read, questions,
completed, seconds, degradations per pin), and marks the task done via
reconciler.mark_done. No result, no files read, or questions == 0 fails
loudly and leaves the run untouched. Refuses a git checkout as source.

Runner image rebuilt and re-pinned (sha256:ebb40a9a...); validation-46
regenerated (digest only). Live, recorded in
benchmarks/measurements/2026-09-30-php-engine-alerts/: froxlor 366 files,
130/130 questions per pin, 6 alerts per pin, 0 in the fix range; wp-gopay
19 files, 101/101 questions, 0 alerts. The loop: 32 run rows (coverage 8,
php=engine), 0 proposals, gate no_proposals, $0.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 13:44:32 +02:00
co-authored by Claude Opus 5.5
parent f4bd984fda
commit 45d02ecf20
35 changed files with 1949 additions and 142 deletions
+32
View File
@@ -99,6 +99,38 @@ def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
COVERAGE_SUMMARY = {
"status": "done", "pins": {"vulnerable": "0" * 40, "fixed": "f" * 40},
"coverage": {
"javascript": {"coverage": "quick", "files": {"vulnerable": 2}},
"php": {"coverage": "none", "files": {"vulnerable": 9, "fixed": 9}},
},
} # fmt: skip
def test_the_alerts_coverage_becomes_one_row_per_language_and_pin() -> None:
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts=[], alerts_summary=COVERAGE_SUMMARY)
got = sorted((r["language"], r["pin_role"], r["pin"], r["coverage"], r["files"]) for r in rows if r["kind"] == "coverage")
assert got == [
("javascript", "vulnerable", "0" * 40, "quick", 2), ("php", "fixed", "f" * 40, "none", 9),
("php", "vulnerable", "0" * 40, "none", 9),
] # fmt: skip
assert rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts_summary={"status": "done"}) == [], "unknown stays unknown"
def test_remember_run_carries_the_alerts_coverage_into_the_store(run_dir: Path, tmp_path: Path) -> None:
(run_dir / f"{CASE}-alerts").mkdir()
(run_dir / f"{CASE}-alerts" / "alerts.jsonl").write_text("") # zero alerts: meaningful only through the coverage
summary = {**COVERAGE_SUMMARY, "pins": {"vulnerable": PIN, "fixed": "f" * 40}}
(run_dir / f"{CASE}-alerts" / "summary.json").write_text(json.dumps(summary))
store = FileStore(tmp_path / "memory")
[result] = remember_run(run_dir, store)
assert result.kinds["coverage"] == 3
php = sorted((r["pin"], r["coverage"]) for r in _by_kind(store)["coverage"] if r["language"] == "php")
assert php == [(PIN, "none"), ("f" * 40, "none")]
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
task = next(d for d in docs if d["kind"] == "Task")