feat(plane): honest coverage in alerts; PHP alerts from the engine on the host

Quick mode has rules only for c/cpp, groovy, java, javascript/typescript and
python, so every PHP case's "0 alerts" read as clean. The `alerts` task now
records per language of the files read whether quick mode covers it
(derived from the bundled ruleset, not a list): coverage quick | none, with
file counts per pin, and `uncovered`. `remember` turns that into `coverage`
memory rows (a new kind; the Run passes the alerts summary.json as input),
loop-measure reports coverage per language, and improve/memory.py's M1/M2
skip a repository+pin whose coverage for the rule's language is only
`none` (counted under `coverage_none` in the proposal's excluded).

`ousast plane alerts-engine <Run.yaml> --source <frozen export>`
(plane/engine_alerts.py) does the alerts task on the host for cases whose
files include a language the engine covers but quick mode does not (PHP):
exports both pins from the case cache, runs the quick scan and the
openultrasast:dev engine container (finding_dump.py, --network none,
--memory 3g, one at a time, per-pin deadline), writes engine findings in
the alerts.jsonl row shape (rule_id engine:<family>, source engine,
in_fix_range from case.json) with a summary (files read, questions,
completed, seconds, degradations per pin), and marks the task done via
reconciler.mark_done. No result, no files read, or questions == 0 fails
loudly and leaves the run untouched. Refuses a git checkout as source.

Runner image rebuilt and re-pinned (sha256:ebb40a9a...); validation-46
regenerated (digest only). Live, recorded in
benchmarks/measurements/2026-09-30-php-engine-alerts/: froxlor 366 files,
130/130 questions per pin, 6 alerts per pin, 0 in the fix range; wp-gopay
19 files, 101/101 questions, 0 alerts. The loop: 32 run rows (coverage 8,
php=engine), 0 proposals, gate no_proposals, $0.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 13:44:32 +02:00
co-authored by Claude Opus 5.5
parent f4bd984fda
commit 45d02ecf20
35 changed files with 1949 additions and 142 deletions
+34
View File
@@ -97,6 +97,39 @@ def test_alerts_attribute_functions_and_mark_the_fix_on_both_pins(tmp_path: Path
assert "an unread tree is not a clean one" in json.loads((out / "summary.json").read_text())["reason"]
def test_quick_coverage_is_derived_from_the_ruleset_directory(tmp_path: Path) -> None:
shipped = alerts.quick_languages()
assert {"python", "javascript", "typescript", "java", "c", "groovy"} <= shipped and "php" not in shipped
assert "php" in alerts.engine_languages(), "the engine's semantic models cover PHP"
rules = tmp_path / "ruleset" / "php"
rules.mkdir(parents=True)
rule = 'rule_id = "php-x"\ntitle = "x"\nlanguages = ["php"]\ncwe = "CWE-89"\ntags = []\npattern = "x"\n'
(rules / "rules.toml").write_text(f"[[rule]]\n{rule}")
assert alerts.quick_languages(tmp_path / "ruleset") == {"php"}, "a ruleset added for a language covers it"
(rules / "rules.toml").write_text(f'[[rule]]\n{rule}status = "disabled"\n')
assert alerts.quick_languages(tmp_path / "ruleset") == set(), "a disabled rule fires nowhere"
def test_alerts_on_an_uncovered_language_record_coverage_none_not_a_clean_zero(tmp_path: Path) -> None:
for side in ("v", "f"):
(tmp_path / side).mkdir()
(tmp_path / side / "index.php").write_text('<?php\n$id = $_GET["id"];\nmysqli_query($c, "SELECT " . $id);\n')
(tmp_path / side / "app.js").write_text("module.exports = 1;\n")
out = tmp_path / "out"
env = {
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_WORKSPACE_DIR": str(tmp_path / "v"), "OUSAST_FIXED_DIR": str(tmp_path / "f"),
"OUSAST_VULNERABLE_PIN": VULNERABLE, "OUSAST_FIXED_PIN": FIXED,
} # fmt: skip
assert alerts.main(env) == 0
summary = json.loads((out / "summary.json").read_text())
assert (out / "alerts.jsonl").read_text() == "" and summary["alerts"] == {"vulnerable": 0, "fixed": 0}
assert summary["coverage"] == {
"javascript": {"coverage": "quick", "files": {"fixed": 1, "vulnerable": 1}},
"php": {"coverage": "none", "files": {"fixed": 1, "vulnerable": 1}},
}
assert summary["uncovered"] == ["php"] and summary["pins"] == {"vulnerable": VULNERABLE, "fixed": FIXED}
def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None:
repo = tmp_path / "r"
repo.mkdir()
@@ -129,6 +162,7 @@ def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path)
names = [e.name for e in run.tasks]
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS]
assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl"
assert run.task(f"{CASE}-remember").inputs["alerts_summary"] == f"{CASE}-alerts/summary.json", "the coverage travels too"
measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"}
assert run.task("loop-measure").inputs == measure_inputs
assert run.task("loop-improve").producers == ("loop-propose",)