mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(plane): honest coverage in alerts; PHP alerts from the engine on the host
Quick mode has rules only for c/cpp, groovy, java, javascript/typescript and python, so every PHP case's "0 alerts" read as clean. The `alerts` task now records per language of the files read whether quick mode covers it (derived from the bundled ruleset, not a list): coverage quick | none, with file counts per pin, and `uncovered`. `remember` turns that into `coverage` memory rows (a new kind; the Run passes the alerts summary.json as input), loop-measure reports coverage per language, and improve/memory.py's M1/M2 skip a repository+pin whose coverage for the rule's language is only `none` (counted under `coverage_none` in the proposal's excluded). `ousast plane alerts-engine <Run.yaml> --source <frozen export>` (plane/engine_alerts.py) does the alerts task on the host for cases whose files include a language the engine covers but quick mode does not (PHP): exports both pins from the case cache, runs the quick scan and the openultrasast:dev engine container (finding_dump.py, --network none, --memory 3g, one at a time, per-pin deadline), writes engine findings in the alerts.jsonl row shape (rule_id engine:<family>, source engine, in_fix_range from case.json) with a summary (files read, questions, completed, seconds, degradations per pin), and marks the task done via reconciler.mark_done. No result, no files read, or questions == 0 fails loudly and leaves the run untouched. Refuses a git checkout as source. Runner image rebuilt and re-pinned (sha256:ebb40a9a...); validation-46 regenerated (digest only). Live, recorded in benchmarks/measurements/2026-09-30-php-engine-alerts/: froxlor 366 files, 130/130 questions per pin, 6 alerts per pin, 0 in the fix range; wp-gopay 19 files, 101/101 questions, 0 alerts. The loop: 32 run rows (coverage 8, php=engine), 0 proposals, gate no_proposals, $0. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
f4bd984fda
commit
45d02ecf20
+5
@@ -0,0 +1,5 @@
|
||||
<?php
|
||||
$id = $_GET["id"];
|
||||
$r = mysqli_query($conn, "SELECT * FROM users WHERE id = " . $id);
|
||||
echo $_GET["name"];
|
||||
system("ping " . $_GET["host"]);
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"root": "/case",
|
||||
"families": [
|
||||
"injection"
|
||||
],
|
||||
"seconds": 58.6,
|
||||
"questions": 1,
|
||||
"completed": 1,
|
||||
"completed_regions": [
|
||||
"app/index.php:"
|
||||
],
|
||||
"degradations": [],
|
||||
"findings": [
|
||||
{
|
||||
"site": "app/index.php:3:<global>",
|
||||
"family": "injection",
|
||||
"rung": "model_entailed",
|
||||
"witness": "$_GET[\"id\"] -> mysqli_query($conn, \"SELECT * FROM users WHERE id = \" . $id) in app/index.php (line 3, 4 steps)",
|
||||
"reached_from": 1
|
||||
},
|
||||
{
|
||||
"site": "app/index.php:5:<global>",
|
||||
"family": "injection",
|
||||
"rung": "model_entailed",
|
||||
"witness": "$_GET[\"host\"] -> system(\"ping \" . $_GET[\"host\"]) in app/index.php (line 5, 2 steps)",
|
||||
"reached_from": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
read 1 files, 145 bytes; 1 regions offer ('injection',)
|
||||
{"root": "/case", "families": ["injection"], "seconds": 58.6, "questions": 1, "completed": 1, "degradations": [], "findings": 2}
|
||||
@@ -149,6 +149,34 @@ def test_m2_stays_silent_on_a_fixed_pin_alert_or_rejected_candidates() -> None:
|
||||
assert propose([*site_hits(), *rejected], rules) == []
|
||||
|
||||
|
||||
# ---- coverage: no rule could fire, so no evidence either way -------------------------------------------------------
|
||||
|
||||
|
||||
def covered(repo: str, pin: str, kind: str, language: str = "python", role: str = "vulnerable") -> dict[str, Any]:
|
||||
return _row("coverage", repo, pin, f"{language}:{role}:{kind}", language=language, coverage=kind, files=3, pin_role=role)
|
||||
|
||||
|
||||
def test_m1_skips_a_repository_and_pin_whose_coverage_for_the_rules_language_is_none() -> None:
|
||||
blind = [*false_alerts(), covered("r2", P["a"], "none")]
|
||||
assert propose(blind, [_rule("noisy")]) == [], "r2's alert could not be evidence: one repository is left"
|
||||
wider = [*false_alerts(repos=("r1", "r1", "r2", "r3")), covered("r3", P["a"], "none")]
|
||||
[proposal] = propose(wider, [_rule("noisy")])
|
||||
assert proposal.provenance["counts"] == {"false_alerts": 3, "repos": 2, "agreed_hits": 0}
|
||||
assert proposal.provenance["excluded"]["coverage_none"] == {f"github.com/o/r3@{P['a']}": 1}
|
||||
|
||||
|
||||
def test_coverage_none_of_another_language_or_beside_engine_coverage_is_no_skip() -> None:
|
||||
other = [*false_alerts(), covered("r2", P["a"], "none", language="php")]
|
||||
assert len(propose(other, [_rule("noisy")])) == 1, "the rule's language (python) was covered there"
|
||||
engine = [*false_alerts(), covered("r2", P["a"], "none"), covered("r2", P["a"], "engine")]
|
||||
assert len(propose(engine, [_rule("noisy")])) == 1, "a later run covered the pin: its alerts are evidence"
|
||||
|
||||
|
||||
def test_m2_skips_a_blind_repository_and_pin() -> None:
|
||||
rules = [_rule("quiet", status="shadow"), _rule("other")]
|
||||
assert propose([*site_hits(), covered("r2", P["b"], "none")], rules) == []
|
||||
|
||||
|
||||
# ---- M3: advisory only ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""`ousast plane alerts-engine` (harnessx-removal Req 6.3): PHP alerts from the engine image, produced on the host.
|
||||
|
||||
No container runs here: the docker runner is injected. The engine's output is the recorded probe
|
||||
(``fixtures/plane-engine/probe.json``: 1 file read, 1 question asked and completed, 2 injection findings)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from test_plane_loop import LOOP, _templates, _write
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.plane import engine_alerts
|
||||
from openultrasast.plane.engine_alerts import EngineAlertsError, alerts_engine, engine_rows, parse_site, read_record
|
||||
from openultrasast.plane.generate import CaseInputs, render
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-engine"
|
||||
CASE = "php-probe"
|
||||
RECORD = json.loads((FIXTURE / "probe.json").read_text())
|
||||
LOG = (FIXTURE / "probe.stdout.txt").read_text()
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> str:
|
||||
return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def world(tmp_path: Path) -> dict[str, Any]:
|
||||
"""A case cache with a vulnerable and a fixed commit, a Run with the case's `alerts` task, a frozen source."""
|
||||
repo = tmp_path / "cache" / CASE
|
||||
(repo / "app").mkdir(parents=True)
|
||||
_git(repo, "init", "-q")
|
||||
shutil.copy(FIXTURE / "app" / "index.php", repo / "app" / "index.php")
|
||||
_git(repo, "add", ".")
|
||||
_git(repo, "-c", "user.name=t", "-c", "[email protected]", "commit", "-qm", "vulnerable")
|
||||
vulnerable = _git(repo, "rev-parse", "HEAD")
|
||||
text = (repo / "app" / "index.php").read_text().replace('" . $id', '" . intval($id)')
|
||||
(repo / "app" / "index.php").write_text(text)
|
||||
_git(repo, "-c", "user.name=t", "-c", "[email protected]", "commit", "-qam", "fixed")
|
||||
fixed = _git(repo, "rev-parse", "HEAD")
|
||||
record = {"id": CASE, "repo": "https://example.com/reserved/probe", "family": "injection", "vulnerable": vulnerable, "fixed": fixed}
|
||||
candidate = ("app/index.php", "<global>", 3)
|
||||
case = CaseInputs(
|
||||
record, (candidate,), (candidate,), {}, 0.0, ranges={"app/index.php": [(3, 3)]}, fixed_ranges={"app/index.php": [(3, 3)]}
|
||||
)
|
||||
files = render([case], _templates(), "validation-46", "t", population="p", split="s", loop=LOOP)
|
||||
source = tmp_path / "frozen"
|
||||
(source / "src" / "openultrasast").mkdir(parents=True)
|
||||
(source / "benchmarks" / "push").mkdir(parents=True)
|
||||
(source / "benchmarks" / "push" / "finding_dump.py").write_text("")
|
||||
run = _write(tmp_path / "plane", files)
|
||||
return {"run": run, "source": source, "cache": tmp_path / "cache", "pins": (vulnerable, fixed), "results": tmp_path / "results"}
|
||||
|
||||
|
||||
class FakeDocker:
|
||||
"""Answers `docker image inspect` and writes ``record`` as the engine's result for every `docker run`."""
|
||||
|
||||
def __init__(self, record: dict[str, Any] | None = RECORD, log: str = LOG) -> None:
|
||||
self.record, self.log, self.runs = record, log, [] # type: ignore[var-annotated]
|
||||
|
||||
def __call__(self, command: Sequence[str], timeout: float) -> subprocess.CompletedProcess[str]:
|
||||
command = list(command)
|
||||
if command[:3] == ["docker", "image", "inspect"]:
|
||||
return subprocess.CompletedProcess(command, 0, "sha256:feed\n", "")
|
||||
mounts = [command[i + 1] for i, part in enumerate(command) if part == "-v"]
|
||||
checkout = Path(next(m for m in mounts if m.endswith(":/case:ro")).split(":")[0])
|
||||
assert (checkout / "app" / "index.php").is_file(), "the pin was exported before the container ran"
|
||||
self.runs.append({"command": command, "timeout": timeout})
|
||||
if self.record is not None:
|
||||
out = Path(next(m for m in mounts if m.endswith(":/out")).split(":")[0])
|
||||
(out / command[command.index("--out") + 1].removeprefix("/out/")).write_text(json.dumps(self.record))
|
||||
return subprocess.CompletedProcess(command, 0, self.log, "")
|
||||
|
||||
|
||||
def _produce(world: dict[str, Any], docker: FakeDocker, tmp_path: Path) -> list[dict[str, Any]]:
|
||||
return alerts_engine(
|
||||
world["run"], source=world["source"], work=tmp_path / "work", cache=world["cache"], deadline=120, runner=docker,
|
||||
results_root=world["results"],
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
def test_recorded_engine_findings_become_alert_rows() -> None:
|
||||
assert parse_site("app/index.php:3:<global>") == ("app/index.php", 3, "<global>")
|
||||
assert parse_site("lib/A.php:76:Foo::bar") == ("lib/A.php", 76, "Foo::bar")
|
||||
record = read_record(FIXTURE / "probe.json", LOG, 0)
|
||||
assert (record["files"], record["bytes"], record["questions"], record["completed"]) == (1, 145, 1, 1)
|
||||
rows = engine_rows(record, FIXTURE, "vulnerable", "a" * 40, {"app/index.php": [[3, 3]]})
|
||||
assert rows == [
|
||||
{"rule_id": "engine:injection", "rule_status": "enabled", "path": "app/index.php", "line": line, "function": "<global>",
|
||||
"pin_role": "vulnerable", "pin": "a" * 40, "in_fix_range": line == 3, "source": "engine"}
|
||||
for line in (3, 5)
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def test_an_engine_that_read_nothing_or_asked_nothing_fails(tmp_path: Path) -> None:
|
||||
out = tmp_path / "x.json"
|
||||
with pytest.raises(EngineAlertsError, match="wrote no result"):
|
||||
read_record(out, "boom", 125)
|
||||
out.write_text(json.dumps({**RECORD, "questions": 0}))
|
||||
with pytest.raises(EngineAlertsError, match="no question was asked"):
|
||||
read_record(out, LOG, 0)
|
||||
out.write_text(json.dumps(RECORD))
|
||||
with pytest.raises(EngineAlertsError, match="did not report reading"):
|
||||
read_record(out, "", 0)
|
||||
|
||||
|
||||
def test_alerts_engine_writes_the_alerts_task_outputs_and_marks_it_done(world: dict[str, Any], tmp_path: Path) -> None:
|
||||
docker = FakeDocker()
|
||||
[result] = _produce(world, docker, tmp_path)
|
||||
assert result["status"] == "done" and len(docker.runs) == 2, "one container per pin, one after the other"
|
||||
for run in docker.runs:
|
||||
command = run["command"]
|
||||
assert command[command.index("--memory") + 1] == "3g" and command[command.index("--network") + 1] == "none"
|
||||
assert command[command.index("--families") + 1] == "injection" and run["timeout"] == 120 + engine_alerts.GRACE
|
||||
out = world["results"] / "validation-46" / f"{CASE}-alerts"
|
||||
rows = [json.loads(line) for line in (out / "alerts.jsonl").read_text().splitlines()]
|
||||
vulnerable, fixed = world["pins"]
|
||||
got = [(r["pin_role"], r["pin"], r["line"], r["in_fix_range"], r["source"]) for r in rows]
|
||||
assert got == [("vulnerable", vulnerable, 3, True, "engine"), ("vulnerable", vulnerable, 5, False, "engine"),
|
||||
("fixed", fixed, 3, True, "engine"), ("fixed", fixed, 5, False, "engine")] # fmt: skip
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (summary["status"], summary["units_done"], summary["units_total"], summary["source"]) == ("done", 2, 2, "engine")
|
||||
assert summary["engine"]["vulnerable"] == {
|
||||
"files": 1, "bytes": 145, "questions": 1, "completed": 1, "seconds": 58.6, "degradations": [], "exit": 0, "findings": 2
|
||||
} # fmt: skip
|
||||
assert summary["coverage"] == {"php": {"coverage": "engine", "files": {"fixed": 1, "vulnerable": 1}}} and summary["uncovered"] == []
|
||||
assert summary["in_fix_range"] == {"vulnerable": 1, "fixed": 1} and summary["image"] == "sha256:feed"
|
||||
assert isinstance(summary["seconds"], float)
|
||||
state = json.loads((world["results"] / "validation-46" / "state.json").read_text())["tasks"][f"{CASE}-alerts"]
|
||||
assert state["status"] == "done" and state["reused"]["source"] == "engine", "the reconciler skips it like reused facts"
|
||||
assert not list((tmp_path / "work").glob(f"{CASE}--*/"))
|
||||
|
||||
|
||||
def test_a_failed_engine_leaves_the_run_untouched(world: dict[str, Any], tmp_path: Path) -> None:
|
||||
with pytest.raises(EngineAlertsError, match="no question was asked"):
|
||||
_produce(world, FakeDocker({**RECORD, "questions": 0, "findings": []}), tmp_path)
|
||||
with pytest.raises(EngineAlertsError, match="wrote no result"):
|
||||
_produce(world, FakeDocker(None), tmp_path)
|
||||
assert not (world["results"] / "validation-46").exists(), "no alerts written, nothing marked done"
|
||||
|
||||
|
||||
def test_the_source_must_be_a_frozen_export(world: dict[str, Any], tmp_path: Path) -> None:
|
||||
(world["source"] / ".git").mkdir()
|
||||
with pytest.raises(EngineAlertsError, match="export it first"):
|
||||
_produce(world, FakeDocker(), tmp_path)
|
||||
|
||||
|
||||
def test_the_cli_reports_and_fails_loudly(world: dict[str, Any], tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str]) -> None: # fmt: skip
|
||||
monkeypatch.setattr(engine_alerts, "run_command", FakeDocker({**RECORD, "questions": 0}))
|
||||
argv = ["plane", "alerts-engine", str(world["run"]), "--source", str(world["source"]), "--repos", str(world["cache"])]
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "r"))
|
||||
assert cli.main(argv) == 2
|
||||
assert "no question was asked" in capsys.readouterr().err
|
||||
@@ -97,6 +97,39 @@ def test_alerts_attribute_functions_and_mark_the_fix_on_both_pins(tmp_path: Path
|
||||
assert "an unread tree is not a clean one" in json.loads((out / "summary.json").read_text())["reason"]
|
||||
|
||||
|
||||
def test_quick_coverage_is_derived_from_the_ruleset_directory(tmp_path: Path) -> None:
|
||||
shipped = alerts.quick_languages()
|
||||
assert {"python", "javascript", "typescript", "java", "c", "groovy"} <= shipped and "php" not in shipped
|
||||
assert "php" in alerts.engine_languages(), "the engine's semantic models cover PHP"
|
||||
rules = tmp_path / "ruleset" / "php"
|
||||
rules.mkdir(parents=True)
|
||||
rule = 'rule_id = "php-x"\ntitle = "x"\nlanguages = ["php"]\ncwe = "CWE-89"\ntags = []\npattern = "x"\n'
|
||||
(rules / "rules.toml").write_text(f"[[rule]]\n{rule}")
|
||||
assert alerts.quick_languages(tmp_path / "ruleset") == {"php"}, "a ruleset added for a language covers it"
|
||||
(rules / "rules.toml").write_text(f'[[rule]]\n{rule}status = "disabled"\n')
|
||||
assert alerts.quick_languages(tmp_path / "ruleset") == set(), "a disabled rule fires nowhere"
|
||||
|
||||
|
||||
def test_alerts_on_an_uncovered_language_record_coverage_none_not_a_clean_zero(tmp_path: Path) -> None:
|
||||
for side in ("v", "f"):
|
||||
(tmp_path / side).mkdir()
|
||||
(tmp_path / side / "index.php").write_text('<?php\n$id = $_GET["id"];\nmysqli_query($c, "SELECT " . $id);\n')
|
||||
(tmp_path / side / "app.js").write_text("module.exports = 1;\n")
|
||||
out = tmp_path / "out"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_WORKSPACE_DIR": str(tmp_path / "v"), "OUSAST_FIXED_DIR": str(tmp_path / "f"),
|
||||
"OUSAST_VULNERABLE_PIN": VULNERABLE, "OUSAST_FIXED_PIN": FIXED,
|
||||
} # fmt: skip
|
||||
assert alerts.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (out / "alerts.jsonl").read_text() == "" and summary["alerts"] == {"vulnerable": 0, "fixed": 0}
|
||||
assert summary["coverage"] == {
|
||||
"javascript": {"coverage": "quick", "files": {"fixed": 1, "vulnerable": 1}},
|
||||
"php": {"coverage": "none", "files": {"fixed": 1, "vulnerable": 1}},
|
||||
}
|
||||
assert summary["uncovered"] == ["php"] and summary["pins"] == {"vulnerable": VULNERABLE, "fixed": FIXED}
|
||||
|
||||
|
||||
def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "r"
|
||||
repo.mkdir()
|
||||
@@ -129,6 +162,7 @@ def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path)
|
||||
names = [e.name for e in run.tasks]
|
||||
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS]
|
||||
assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl"
|
||||
assert run.task(f"{CASE}-remember").inputs["alerts_summary"] == f"{CASE}-alerts/summary.json", "the coverage travels too"
|
||||
measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"}
|
||||
assert run.task("loop-measure").inputs == measure_inputs
|
||||
assert run.task("loop-improve").producers == ("loop-propose",)
|
||||
|
||||
@@ -99,6 +99,38 @@ def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
|
||||
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
|
||||
|
||||
|
||||
COVERAGE_SUMMARY = {
|
||||
"status": "done", "pins": {"vulnerable": "0" * 40, "fixed": "f" * 40},
|
||||
"coverage": {
|
||||
"javascript": {"coverage": "quick", "files": {"vulnerable": 2}},
|
||||
"php": {"coverage": "none", "files": {"vulnerable": 9, "fixed": 9}},
|
||||
},
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def test_the_alerts_coverage_becomes_one_row_per_language_and_pin() -> None:
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts=[], alerts_summary=COVERAGE_SUMMARY)
|
||||
got = sorted((r["language"], r["pin_role"], r["pin"], r["coverage"], r["files"]) for r in rows if r["kind"] == "coverage")
|
||||
assert got == [
|
||||
("javascript", "vulnerable", "0" * 40, "quick", 2), ("php", "fixed", "f" * 40, "none", 9),
|
||||
("php", "vulnerable", "0" * 40, "none", 9),
|
||||
] # fmt: skip
|
||||
assert rows_for(ctx, facts=None, passes={}, models={}, agreed=None, alerts_summary={"status": "done"}) == [], "unknown stays unknown"
|
||||
|
||||
|
||||
def test_remember_run_carries_the_alerts_coverage_into_the_store(run_dir: Path, tmp_path: Path) -> None:
|
||||
(run_dir / f"{CASE}-alerts").mkdir()
|
||||
(run_dir / f"{CASE}-alerts" / "alerts.jsonl").write_text("") # zero alerts: meaningful only through the coverage
|
||||
summary = {**COVERAGE_SUMMARY, "pins": {"vulnerable": PIN, "fixed": "f" * 40}}
|
||||
(run_dir / f"{CASE}-alerts" / "summary.json").write_text(json.dumps(summary))
|
||||
store = FileStore(tmp_path / "memory")
|
||||
[result] = remember_run(run_dir, store)
|
||||
assert result.kinds["coverage"] == 3
|
||||
php = sorted((r["pin"], r["coverage"]) for r in _by_kind(store)["coverage"] if r["language"] == "php")
|
||||
assert php == [(PIN, "none"), ("f" * 40, "none")]
|
||||
|
||||
|
||||
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
|
||||
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
|
||||
task = next(d for d in docs if d["kind"] == "Task")
|
||||
|
||||
Reference in New Issue
Block a user