feat(pre-push-safety-net): preserve exact ranker scope and outcomes

This commit is contained in:
Marius Ciepluch
2026-09-13 10:16:22 +02:00
parent bd53b6dc9d
commit 546db24b8e
10 changed files with 931 additions and 37 deletions
+7
View File
@@ -90,6 +90,13 @@ To exercise the same smoke through one shared backend deadline, add
all census/witness queries consume that same budget. The generous smoke allowance verifies
runtime integration; it does not represent the hook's target latency.
`ops/smoke_scope.py` exercises the generic driver with real PHP and JavaScript graphs,
explicit evidence ranking and two declared entry regions per language. Run it with the
same Docker command, substituting `smoke_scope.py` for `smoke_engine.py`. It verifies
that the lower static-ranked risky function is selected, that the harmless function is
recorded as deferred, and that selected IDs match completed query outcomes and per-family
coverage. It uses one 900-second lab budget; it is not a mapper or hook-quality evaluation.
Why this exists: Joern was first installed here by hand with `curl` and `unzip`, which is fine once and
unreproducible thereafter. The engine decides whether a finding is a `suspicion` or a `model_entailed`, and
every ceiling number in `benchmarks/measurements` is only comparable against a fixed version — so the version
+72
View File
@@ -0,0 +1,72 @@
"""Real engine scope smoke with declared entry regions; not a hook quality benchmark."""
from __future__ import annotations
import hashlib
import json
import tempfile
import time
from dataclasses import asdict
from pathlib import Path
from openultrasast.cpg.backend import JoernBackend
from openultrasast.model.contracts import ExecutionBudget
from openultrasast.model.regions import ScanRegion
from openultrasast.model.scan import ScanBudget, scan_repository
def main() -> None:
budget = ExecutionBudget(time.monotonic() + 900, 2.0)
report: dict[str, object] = {
"purpose": "Real PHP/JavaScript scope smoke; declared entry regions, no admission or latency claim",
"shared_budget_seconds": 900,
"languages": {},
}
languages = report["languages"]
assert isinstance(languages, dict)
with tempfile.TemporaryDirectory(prefix="ousast-scope-smoke-") as temporary:
for language, filename, content in (
("php", "api.php", "<?php\nfunction safe($req) { return 42; }\nfunction risky($req) { eval($req); }\n"),
("javascript", "api.js", "function safe(req) { return 42; }\nfunction risky(req) { eval(req); }\n"),
):
root = Path(temporary) / language
root.mkdir()
source = root / filename
source.write_text(content)
data = source.read_bytes()
assert data and data == content.encode(), "source input was not read"
regions = tuple(
ScanRegion(filename, function, language, ("injection",), rank, "entry_point")
for function, rank in (("safe", 2.0), ("risky", 1.0))
)
result = scan_repository(
root,
regions,
backend=JoernBackend(heap_mb=1024),
budget=ScanBudget(max_regions=1),
execution_budget=budget,
ranking_mode="evidence",
unit="scope-smoke",
population_complete=True,
)
scope = result.scope
assert scope is not None, "no scope decision"
assert [item.identity.function for item in scope.selected] == ["risky"], scope
assert [item.identity.function for item in scope.deferred] == ["safe"], scope
selected_ids = [item.identity.question_id for item in scope.selected]
assert [item.identity.question_id for item in result.question_outcomes] == selected_ids
assert all(item.status == "completed" and item.raw_rows_json is not None for item in result.question_outcomes), result
assert len(result.family_coverage) == 1
coverage = result.family_coverage[0]
assert (coverage.selected, coverage.completed, coverage.unanswered, coverage.deferred) == (1, 1, 0, 1), coverage
languages[language] = {
"source_bytes": len(data),
"source_sha256": hashlib.sha256(data).hexdigest(),
"selected_question_ids": selected_ids,
"result": asdict(result),
}
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()