mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
feat(plane): honest coverage in alerts; PHP alerts from the engine on the host
Quick mode has rules only for c/cpp, groovy, java, javascript/typescript and python, so every PHP case's "0 alerts" read as clean. The `alerts` task now records per language of the files read whether quick mode covers it (derived from the bundled ruleset, not a list): coverage quick | none, with file counts per pin, and `uncovered`. `remember` turns that into `coverage` memory rows (a new kind; the Run passes the alerts summary.json as input), loop-measure reports coverage per language, and improve/memory.py's M1/M2 skip a repository+pin whose coverage for the rule's language is only `none` (counted under `coverage_none` in the proposal's excluded). `ousast plane alerts-engine <Run.yaml> --source <frozen export>` (plane/engine_alerts.py) does the alerts task on the host for cases whose files include a language the engine covers but quick mode does not (PHP): exports both pins from the case cache, runs the quick scan and the openultrasast:dev engine container (finding_dump.py, --network none, --memory 3g, one at a time, per-pin deadline), writes engine findings in the alerts.jsonl row shape (rule_id engine:<family>, source engine, in_fix_range from case.json) with a summary (files read, questions, completed, seconds, degradations per pin), and marks the task done via reconciler.mark_done. No result, no files read, or questions == 0 fails loudly and leaves the run untouched. Refuses a git checkout as source. Runner image rebuilt and re-pinned (sha256:ebb40a9a...); validation-46 regenerated (digest only). Live, recorded in benchmarks/measurements/2026-09-30-php-engine-alerts/: froxlor 366 files, 130/130 questions per pin, 6 alerts per pin, 0 in the fix range; wp-gopay 19 files, 101/101 questions, 0 alerts. The loop: 32 run rows (coverage 8, php=engine), 0 proposals, gate no_proposals, $0. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f4bd984fda
commit
45d02ecf20
@@ -19,6 +19,11 @@ final counts as no evidence in either direction.
|
||||
- **M3, disputed families**: families with at least :data:`M3_MIN_CANDIDATES` candidates and a dispute rate of at
|
||||
least :data:`M3_MIN_DISPUTE_RATE`. Advisory (``signals.json``); never an edit, never shown to the validator.
|
||||
|
||||
**Coverage**: M1 and M2 skip every repository and pin whose ``coverage`` rows record the rule's language only as
|
||||
``none`` (quick mode had no rule for it and no engine alerts were produced): a rule cannot be demoted or promoted
|
||||
on evidence that could not exist. The skipped alerts are counted per ``repo@pin`` under ``coverage_none`` in the
|
||||
proposal's ``excluded``.
|
||||
|
||||
**Train-on-test guard** (:class:`Guard`), applied before any rule sees a row: a row is dropped when its population
|
||||
is one the proposal will be qualified on, its (repo, pin) is a case of the gated benchmark manifest, or its
|
||||
repository is a holdout pair's repository. The pair catalog records a holdout pair's *fix* commit, not the
|
||||
@@ -142,6 +147,7 @@ class _View:
|
||||
verdicts: dict[tuple[str, str, str], Record] = field(default_factory=dict) # (repo, pin, candidate)
|
||||
sites: set[tuple[str, str]] = field(default_factory=set) # (repo, candidate) with site_match true
|
||||
alerts: dict[str, list[Record]] = field(default_factory=dict) # rule_id -> alerts
|
||||
coverage: dict[tuple[str, str, str], set[str]] = field(default_factory=dict) # (repo, pin, language) -> recorded kinds
|
||||
|
||||
@classmethod
|
||||
def of(cls, records: Sequence[Record]) -> _View:
|
||||
@@ -151,7 +157,9 @@ class _View:
|
||||
seen: set[tuple[object, ...]] = set()
|
||||
for rec in records:
|
||||
row = rec.row
|
||||
if row["kind"] == "verdict":
|
||||
if row["kind"] == "coverage" and row.get("language"):
|
||||
view.coverage.setdefault((row["repo"], row["pin"], str(row["language"])), set()).add(str(row.get("coverage")))
|
||||
elif row["kind"] == "verdict":
|
||||
view.verdicts[(row["repo"], row["pin"], str(row.get("candidate")))] = rec
|
||||
if row.get("site_match") is True:
|
||||
view.sites.add((row["repo"], str(row.get("candidate"))))
|
||||
@@ -171,6 +179,26 @@ class _View:
|
||||
flag = alert.get("in_fix_range")
|
||||
return bool(flag) if flag is not None else (alert["repo"], _cand(alert)) in self.sites
|
||||
|
||||
def blind(self, repo: str, pin: str, languages: Iterable[str]) -> bool:
|
||||
"""True when a language of the rule was recorded at this repository and pin with coverage ``none`` only: no
|
||||
rule could fire there, so what the rule did or did not say there is no evidence either way."""
|
||||
return any(self.coverage.get((repo, pin, language)) == {"none"} for language in languages)
|
||||
|
||||
def seeing(self, languages_by_rule: Mapping[str, Sequence[str]]) -> tuple[_View, dict[str, int]]:
|
||||
"""This view without the alerts at a repository and pin blind to their rule's language, and how many were
|
||||
dropped per ``repo@pin``."""
|
||||
kept: dict[str, list[Record]] = {}
|
||||
dropped: dict[str, int] = {}
|
||||
for rule_id, alerts in self.alerts.items():
|
||||
languages = languages_by_rule.get(rule_id, ())
|
||||
for alert in alerts:
|
||||
if self.blind(alert.row["repo"], alert.row["pin"], languages):
|
||||
label = f"{alert.row['repo']}@{alert.row['pin']}"
|
||||
dropped[label] = dropped.get(label, 0) + 1
|
||||
else:
|
||||
kept.setdefault(rule_id, []).append(alert)
|
||||
return _View(self.verdicts, self.sites, kept, self.coverage), dict(sorted(dropped.items()))
|
||||
|
||||
|
||||
def _m1(rule_id: str, view: _View) -> tuple[list[Record], dict[str, int]] | None:
|
||||
evidence: list[Record] = []
|
||||
@@ -262,7 +290,9 @@ def propose_from_memory(
|
||||
) -> list[MemoryProposal]:
|
||||
"""M1 and M2 over the rows the guard keeps, minus proposals already reverted on the same evidence."""
|
||||
kept, dropped = excluded.apply(rows)
|
||||
view = _View.of(kept)
|
||||
view, blind = _View.of(kept).seeing({rid: rule.languages for rid, rule in ruleset_by_id.items()})
|
||||
if blind:
|
||||
dropped = {**dropped, "coverage_none": blind}
|
||||
statuses = {rid: _status(rid, ruleset_by_id, current_ledger) for rid in ruleset_by_id}
|
||||
enabled = {rid for rid, status in statuses.items() if status == "enabled"}
|
||||
proposals: list[MemoryProposal] = []
|
||||
|
||||
Reference in New Issue
Block a user