diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 16db072..bbf806a 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -87,7 +87,7 @@ - _Requirements: 4.1_ - _Depends: 1.3_ -- [ ] 2.2 The first repository measurement +- [x] 2.2 The first repository measurement - Run the full scan against that checkout and commit the artifact: CPG build time, query time and total scan wall-clock as three separate numbers, peak memory, findings by rung, cost, `regions_unjudged`, and **whether the known CVE was found and at which rung**. The build/query split is separated because @@ -118,6 +118,50 @@ - _Requirements: 4.2_ - _Depends: 2.2_ +- [ ] 2.4 A region that spans the flow, not just a function + - **2.2's finding, and the one that reshapes the phase.** VAmPI-SQLI is a request parameter reaching + `get_by_username(username)`, passed to `User.get_user(username)` in another module, interpolated into a + query and executed. Source and sink are in different functions, regions are per-function, and a + repository scan does not treat parameters as untrusted -- so neither region sees both ends and the + engine is silent on the most ordinary shape a real injection bug has. + - The CPG already has the call graph; `reachableByFlows` is not function-bounded. What is bounded is the + QUESTION the driver asks. So the work is in the region model and the query scope, not in the arbiter: + a region should be able to name an entry point and admit sinks reachable from it, with the file scope + from 2.2 kept so the answer is still attributable. + - Watch the cost: an unbounded interprocedural question over a large repository is how this gets slow + again. Measure it on vampi first, then on the envelope checkout. + - Observable: VAmPI-SQLI entailed or corroborated at `models/user_model.py:get_user`, with the scan's + wall-clock and the entailed-finding count recorded against 2.2's numbers. + - _Requirements: 4.4, 8.1_ + - _Depends: 2.2_ + +- [ ] 2.5 Module-level regions the engine can actually ask about + - A setting in `if __name__ == '__main__':` belongs to a region the mapper calls `app.py:__main__`, but + Joern names that method ``. The function filter matches nothing, so module-level configuration + is invisible. Before 2.2's file scoping it was worse, not better: the literal was reported eight times + in eight files that do not contain it. + - Observable: `app.py:17`'s `host='0.0.0.0'` reported exactly once, at that location. + - _Requirements: 4.4_ + - _Depends: 2.2_ + +- [ ] 2.6 Access control that does not entail every public endpoint + - 7 of 9 VAmPI handlers were entailed, including `register_user` and `get_all_users`, which are meant to + be unauthenticated. The dominance rule -- unguarded, with guarded siblings -- was calibrated on pairs + where a guarded twin always existed. At repository scale every public endpoint has guarded siblings. + - This is a precision question and needs a precision measurement: on a real repository, what fraction of + entailed access-control findings are endpoints that genuinely require authorisation? Do not tune it + against VAmPI alone; that is how the +16% overfitting gap in Req 8 happened. + - Observable: a recorded precision figure before and after, on more than one checkout. + - _Requirements: 8.1, 8.2_ + - _Depends: 2.2_ + +- [ ] 2.7 A dominance witness that names its line + - Access-control sites read `api_views/users.py:?:update_password`. Taint and config witnesses carry + `(line N)`; dominance does not, so its findings have no line and a contributor cannot open them. + - Observable: every entailed finding in a repository scan has an integer line. + - _Requirements: 5.1_ + - _Depends: 2.2_ + ## Group 3 — Ship it - [ ] 3.1 Contributor output diff --git a/benchmarks/measurements/2026-09-09-repo-vampi.json b/benchmarks/measurements/2026-09-09-repo-vampi.json new file mode 100644 index 0000000..1a417bf --- /dev/null +++ b/benchmarks/measurements/2026-09-09-repo-vampi.json @@ -0,0 +1,143 @@ +{ + "caveat": "One repository of 520 lines. It measures whether the shape works, not whether the engine is accurate: two known vulnerabilities is not a detection rate, and libpng (the envelope checkout) has not been run.", + "completed": true, + "config": "shipped defaults", + "degradations": [ + { + "detail": "no path records; obligations evaluated within entry-point handlers only", + "files": [ + "api_views/books.py", + "api_views/users.py", + "models/books_model.py", + "models/user_model.py" + ], + "language": "python", + "reason": "obligations_function_local", + "stage": "obligations" + }, + { + "reason": "budget_exhausted", + "regions_unasked": 17, + "stage": "model" + }, + { + "reason": "hunter_model_unavailable", + "stage": "map" + } + ], + "exit_code": 0, + "findings_by_rung": { + "execution_confirmed": 0, + "model_corroborated": 0, + "model_entailed": 7, + "suspicion": 53 + }, + "findings_total": 60, + "generated": "2026-09-09", + "known_outcomes": [ + { + "family": "injection", + "finding_id": "overlay-coverage:models/user_model.py:73:execute", + "id": "VAmPI-SQLI", + "line": 73, + "location": "models/user_model.py:get_user:73", + "matches": 2, + "result": "found", + "rung": "suspicion" + }, + { + "family": "access_control", + "finding_id": "model:access_control:api_views/users.py:?:update_password", + "id": "VAmPI-BOLA", + "line": null, + "location": "api_views/users.py:update_password:187", + "matches": 5, + "result": "found", + "rung": "model_entailed" + } + ], + "mode": "standard", + "model": { + "arbitrate_seconds": 111.6, + "build_seconds": 6.17, + "by_rung": { + "execution_confirmed": 0, + "model_corroborated": 0, + "model_entailed": 7, + "suspicion": 37 + }, + "cost_usd": 0.0067, + "model_calls": 105, + "query_seconds": 40.86, + "regions_scanned": 22, + "regions_unasked": 17, + "regions_unjudged": 0, + "seconds": 158.63, + "unjudged_sample": [] + }, + "open_defects": [ + { + "detail": "app.py's permissive literal sits in . Its region is app.py:__main__, but Joern names the module-level method , so the function filter matches nothing. It was reported eight times in the wrong files before the fix and zero times after: less wrong, still wrong.", + "id": "module-level-regions-are-unaskable" + }, + { + "detail": "VAmPI-SQLI: request parameter -> get_by_username(username) -> User.get_user(username) -> f-string -> execute. Source and sink are in different functions and different modules; regions are per-function and repository scans do not treat parameters as untrusted, so neither region sees both ends. This is the shape most real injection bugs have.", + "id": "region-scoped-taint-cannot-cross-a-function" + }, + { + "detail": "7 of 9 handlers entailed, including register_user and get_all_users, which are meant to be unauthenticated. At pair scale a guarded twin always existed; at repository scale 'unguarded with guarded siblings' fires on every public endpoint.", + "id": "access-control-entails-too-broadly" + }, + { + "detail": "Access-control sites read api_views/users.py:?:update_password -- a location no editor can open.", + "id": "dominance-witnesses-carry-no-line" + } + ], + "peak_rss_mb": 868.6, + "reading": "GO on one CPG per repository, NO-GO on region-scoped taint as the only shape. The engine handles a real checkout comfortably: one CPG for 11 files in 6.2s, every one of 22 regions arbitrated, 177s end to end at 869MB and $0.0067. Nothing here says a repository is infeasible, so phases 3-6 can be built on this shape. What it does say is that the region model has a ceiling: VAmPI-BOLA is entailed because the obligation and its absence live in one function, and VAmPI-SQLI is missed because its source is in the handler and its sink is in a different module. A per-function region cannot see a flow that crosses functions, which is the common real-world shape rather than an edge case.", + "repo": { + "commit": "f16052dce83f05847133ec98f01c5193a41de7d8", + "known": [ + { + "family": "injection", + "id": "VAmPI-SQLI", + "in_scope": true, + "location": "models/user_model.py:get_user:73", + "weakness": "SQL injection: the username is interpolated into the query string at line 72 and executed at 73" + }, + { + "family": "access_control", + "id": "VAmPI-BOLA", + "in_scope": true, + "location": "api_views/users.py:update_password:187", + "weakness": "unauthorized password change: the path username is used instead of the authenticated token subject" + } + ], + "language": "python", + "license": "MIT", + "measures": [ + "detection" + ], + "name": "vampi", + "url": "https://github.com/erev0s/VAmPI.git" + }, + "scan_seconds": 177.09, + "spec": "contributor-scan 2.2 (Req 4.1-4.5)", + "stages": { + "completed": [ + "static", + "map" + ], + "requested": [ + "static", + "map" + ], + "skipped": [] + }, + "what_this_run_changed": { + "budget_no_longer_stops_free_work": "The budget bounds model calls; it was enforced by breaking the region loop, so a paid limit stopped unpaid CPG arbitration. 6 of 22 regions were examined and the BOLA was missed. It now withholds the judge only: 22 of 22 arbitrated, 17 counted as regions_unasked, and the BOLA is entailed.", + "queries_are_scoped_to_a_file": "A region with no enclosing function sent function='' and matched the whole repository, and the driver attributed that answer to whichever region asked: one permissive literal in app.py became eight identical entailed findings in eight files that do not contain it. Queries now carry the region's file. Entailed findings fell 14 -> 7 and query time 70.9s -> 40.9s.", + "the_batch_path_was_unreachable": "query_batch had been implemented, tested and committed while CpgResult exposed only run, so the driver could not discover it and fell back to one JVM per region-family: the first attempt at this measurement ran 18 minutes on 520 lines. The earlier 275s->106s improvement attributed to batching came from region deduplication alone.", + "the_jvm_heap_is_bounded": "Joern with no -Xmx claimed a quarter of physical RAM; runs were killed under memory pressure at different points, so the numbers described the host. Now -J-Xmx2048m by default, OPENULTRASAST_CPG_HEAP_MB to raise it. This run was measured at 1536MB." + } +} diff --git a/benchmarks/repos/measure.py b/benchmarks/repos/measure.py new file mode 100644 index 0000000..a947724 --- /dev/null +++ b/benchmarks/repos/measure.py @@ -0,0 +1,198 @@ +"""contributor-scan 2.2: measure one pinned checkout end to end. + + python benchmarks/repos/measure.py vampi --out benchmarks/measurements/-repo-vampi.json + +This is the go/no-go instrument for the repository shape, so it records the things that decide it rather +than a single verdict: + +* **Where the wall-clock went** -- CPG build, Joern queries, and arbitration as three numbers. A total cannot + say whether a warm Joern server (task 2.3) would buy anything, or whether the ceiling is the engine or the + budget. +* **What went unjudged.** A scan that ran out of budget and a scan that examined everything both report + findings; only the unjudged count separates them. +* **Whether each KNOWN vulnerability was found, and at which rung** -- but only for the ones the recipe marks + in scope. An out-of-family CVE is reported as `out_of_scope`, never as a miss. + +The scan runs against a copy: the pinned checkout stays pristine, and a scan's own artifacts never +accumulate inside the thing being measured. +""" + +from __future__ import annotations + +import argparse +import json +import resource +import shutil +import subprocess +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + +from openultrasast.repos import ( # noqa: E402 + DEFAULT_REPO_DIR, + KnownVulnerability, + load_repo_recipes, + recipe_payload, + resolve, + verify, +) + +# How far from the declared line a finding may land and still be the same defect. A taint finding lands on +# the sink, an obligation finding on the handler, and the two can be tens of lines apart in one function. +LINE_TOLERANCE = 40 +RUNG_ORDER = ("execution_confirmed", "model_entailed", "model_corroborated", "suspicion") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="measure a scan of one pinned known-vulnerable checkout") + parser.add_argument("name") + parser.add_argument("--dir", type=Path, default=DEFAULT_REPO_DIR) + parser.add_argument("--fetch", action="store_true", help="allow network to materialise the checkout") + parser.add_argument("--mode", default="standard", choices=("quick", "standard", "deep")) + parser.add_argument("--work", type=Path, default=None, help="where the scanned copy goes (default: a temp dir)") + parser.add_argument("--config", type=Path, default=None, help="scan config; without one the shipped defaults apply") + parser.add_argument("--out", type=Path, default=None) + args = parser.parse_args(argv) + + recipes = [item for item in load_repo_recipes(args.dir) if item.name == args.name] + if not recipes: + raise SystemExit(f"no recipe named {args.name!r} in {args.dir}") + recipe = recipes[0] + + root = resolve(recipe, fetch=True if args.fetch else None) + problems = verify(recipe, root) + if problems: + raise SystemExit(f"{recipe.name}: the checkout contradicts the recipe: {problems}") + + work = args.work or Path(f"/tmp/ousast-measure-{recipe.name}") # noqa: S108 - operator-visible, overridable + _copy(root, work) + + outcome = _scan(work, args.mode, args.config) + payload = { + "generated": datetime.now(UTC).strftime("%Y-%m-%d"), + "spec": "contributor-scan 2.2 (Req 4.1-4.5)", + "repo": recipe_payload(recipe), + "mode": args.mode, + "config": str(args.config) if args.config else "shipped defaults", + **outcome, + "known_outcomes": [_outcome(item, outcome["findings"]) for item in recipe.known], + } + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if args.out: + args.out.write_text(text) + print(f"wrote {args.out}") + else: + print(text) + return 0 + + +def _copy(root: Path, work: Path) -> None: + if work.exists(): + shutil.rmtree(work) + shutil.copytree(root, work, ignore=shutil.ignore_patterns(".git")) + + +def _scan(work: Path, mode: str, config: Path | None = None) -> dict[str, object]: + """Run the scan as a child process, so its peak RSS is measurable and a crash is a recorded stage.""" + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + started = time.monotonic() + proc = subprocess.run( + [sys.executable, "-m", "openultrasast.cli", "scan", str(work), "--mode", mode] + (["--config", str(config)] if config else []), + capture_output=True, + text=True, + ) + wall = round(time.monotonic() - started, 2) + peak_kb = max(resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss, before) + + runs = sorted((work / ".openultrasast" / "runs").glob("*/")) + manifest = runs[-1] / "manifest.json" if runs else None + if proc.returncode not in (0, 1) or manifest is None or not manifest.is_file(): + # Req 4.3: a repository that cannot be processed records the stage it failed at, not an opaque error. + return { + "completed": False, + "failed_at": "scan" if manifest is None else "manifest", + "exit_code": proc.returncode, + "stderr_tail": proc.stderr[-2000:], + "scan_seconds": wall, + "peak_rss_mb": round(peak_kb / 1024, 1), + "findings": [], + } + + data = json.loads(manifest.read_text()) + model = dict(data.get("model", {})) + findings = _findings(runs[-1]) + return { + "completed": True, + "exit_code": proc.returncode, + "scan_seconds": wall, + "peak_rss_mb": round(peak_kb / 1024, 1), + "model": model, + "stages": data.get("stages", {}), + "degradations": data.get("degradations", []), + "findings_total": len(findings), + "findings_by_rung": _tally(findings), + "findings": findings, + } + + +def _findings(run_dir: Path) -> list[dict[str, object]]: + path = run_dir / "findings.json" + if not path.is_file(): + return [] + data = json.loads(path.read_text()) + rows = data if isinstance(data, list) else data.get("findings", []) + return [dict(row) for row in rows] + + +def _tally(findings: list[dict[str, object]]) -> dict[str, int]: + counts = {rung: 0 for rung in RUNG_ORDER} + for row in findings: + rung = str(row.get("rung", "suspicion")) + counts[rung] = counts.get(rung, 0) + 1 + return counts + + +def _outcome(known: KnownVulnerability, findings: list[dict[str, object]]) -> dict[str, object]: + """Whether this vulnerability was found -- and, when it was never in scope, that it was not asked.""" + if not known.in_scope: + return { + "id": known.id, + "location": known.location, + "family": known.family, + "result": "out_of_scope", + "note": "this engine arbitrates no family that decides this weakness; 'not found' would be a category error, not a miss", + } + matches = [row for row in findings if _matches(known, row)] + if not matches: + return {"id": known.id, "location": known.location, "family": known.family, "result": "missed"} + best = min(matches, key=lambda row: _rank(str(row.get("rung", "suspicion")))) + return { + "id": known.id, + "location": known.location, + "family": known.family, + "result": "found", + "rung": best.get("rung"), + "finding_id": best.get("finding_id"), + "line": best.get("line"), + "matches": len(matches), + } + + +def _matches(known: KnownVulnerability, row: dict[str, object]) -> bool: + if str(row.get("path", "")) != known.file: + return False + if str(row.get("function_name") or "") == known.function: + return True + line = row.get("line") + return isinstance(line, int) and abs(line - known.line) <= LINE_TOLERANCE + + +def _rank(rung: str) -> int: + return RUNG_ORDER.index(rung) if rung in RUNG_ORDER else len(RUNG_ORDER) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 9adb652..3b0c1ee 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -883,10 +883,14 @@ def _model_payload(result: object, *, unjudged_paths: tuple[str, ...] = ()) -> d "by_rung": dict(getattr(result, "by_rung", {}) or {}), "regions_scanned": getattr(result, "regions_scanned", 0), "regions_unjudged": getattr(result, "regions_unjudged", 0), + "regions_unasked": getattr(result, "regions_unasked", 0), "unjudged_sample": list(unjudged_paths), "model_calls": getattr(result, "model_calls", 0), "cost_usd": getattr(result, "cost_usd", 0.0), "seconds": getattr(result, "seconds", 0.0), + "build_seconds": getattr(result, "build_seconds", 0.0), + "query_seconds": getattr(result, "query_seconds", 0.0), + "arbitrate_seconds": getattr(result, "arbitrate_seconds", 0.0), } @@ -952,7 +956,7 @@ def _finding_from_model(item: object) -> StaticFinding: """A model finding as a StaticFinding, carrying its rung and witness (Req 5.1).""" site = str(getattr(item, "site", "")) path, _, rest = site.partition(":") - line = rest.split(":")[0] if rest else "" + line, _, function = rest.partition(":") rung = getattr(item, "rung", None) witness = str(getattr(item, "witness", "") or getattr(item, "contradiction", "")) family = str(getattr(item, "family", "")) @@ -965,7 +969,7 @@ def _finding_from_model(item: object) -> StaticFinding: evidence_level="suspicion", rationale=witness or f"the model layer reported a {family} candidate", line=int(line) if line.isdigit() else None, - function_name=None, + function_name=function or None, reachability_status="unknown", reachability_evidence=[], reachability_conditions=[], diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index 7f8f737..a3ffe6e 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -40,6 +40,12 @@ END = "---OUSAST-CPG-END---" BUILD_TIMEOUT_SECONDS = 900 # a large tree takes minutes; the go/no-go records what it actually cost QUERY_TIMEOUT_SECONDS = 300 +# A JVM with no -Xmx takes a quarter of physical RAM for its heap. On a contributor's laptop that is the +# difference between a scan running in the background and a scan the machine notices, and it made the +# first repository measurements unreproducible: runs were killed under memory pressure at different +# points, so the numbers described the host, not the engine. Bound it, and let the operator raise it. +CPG_HEAP_MB = 2048 +HEAP_ENV = "OPENULTRASAST_CPG_HEAP_MB" Runner = Callable[..., subprocess.CompletedProcess[str]] @@ -61,10 +67,18 @@ def extract_payload(stdout: str) -> object | None: @dataclass(frozen=True) class CpgResult: - """A built CPG and the way to query it. Opaque: only ``model/`` reads through ``run``.""" + """A built CPG and the ways to query it. Opaque: only ``model/`` reads through these. + + ``run_batch`` is optional so a backend that cannot answer many questions in one invocation still fits + this shape -- the driver falls back to ``run``, once per request. But a backend that CAN batch must + attach it HERE, because this is the driver's only way to discover the capability. Leaving it off does + not fail: it silently costs a JVM per region, which is how batching sat unreachable behind a passing + test suite until a repository scan made the cost visible. + """ cpg_path: Path run: Callable[[str, Mapping[str, object]], object | None] + run_batch: Callable[[str, Mapping[str, Mapping[str, object]]], dict[str, list[object]]] | None = None class CpgBackend(Protocol): @@ -94,6 +108,7 @@ class JoernBackend: runner: Runner | None = None build_timeout: int = BUILD_TIMEOUT_SECONDS query_timeout: int = QUERY_TIMEOUT_SECONDS + heap_mb: int = 0 # 0 means read the environment, then fall back to CPG_HEAP_MB queries_dir: Path = field(default_factory=lambda: QUERIES_DIR) def available(self) -> bool: @@ -108,13 +123,17 @@ class JoernBackend: return None scratch = Path(tempfile.mkdtemp(prefix="ousast-cpg-")) cpg_path = scratch / "cpg.bin" - command = [parse or "joern-parse", str(root), "--output", str(cpg_path)] + command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)] completed = self._run(command, timeout=self.build_timeout, cwd=scratch) if completed is None or completed.returncode != 0 or not cpg_path.is_file(): detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout" logger.warning("cpg build failed for %s: %s", root, detail) return None - return CpgResult(cpg_path=cpg_path, run=lambda query, params: self.query(cpg_path, query, params)) + return CpgResult( + cpg_path=cpg_path, + run=lambda query, params: self.query(cpg_path, query, params), + run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests), + ) def query_batch(self, cpg_path: Path, query: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]: """Run ONE script invocation carrying many requests, keyed back to their ids. @@ -135,6 +154,7 @@ class JoernBackend: payload = json.dumps({rid: {k: _render(v) for k, v in req.items()} for rid, req in requests.items()}) command = [ shutil.which("joern") or "joern", + self._heap_flag(), "--script", str(script), "--param", @@ -159,7 +179,7 @@ class JoernBackend: if not script.is_file(): logger.warning("no such cpg query: %s", script) return None - command = [shutil.which("joern") or "joern", "--script", str(script), "--param", f"cpgFile={cpg_path}"] + command = [shutil.which("joern") or "joern", self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"] for key, value in sorted(params.items()): command += ["--param", f"{key}={_render(value)}"] # Joern writes a `workspace/` beside the working directory; run it inside the CPG's own scratch dir so @@ -171,6 +191,13 @@ class JoernBackend: return None return extract_payload(completed.stdout or "") + def _heap_flag(self) -> str: + """``-J-Xmx``: both launchers forward ``-J`` arguments to the JVM.""" + if self.heap_mb > 0: + return f"-J-Xmx{self.heap_mb}m" + configured = os.environ.get(HEAP_ENV, "").strip() + return f"-J-Xmx{configured if configured.isdigit() else CPG_HEAP_MB}m" + def _run(self, command: list[str], *, timeout: int, cwd: Path | None = None) -> subprocess.CompletedProcess[str] | None: run = self.runner if self.runner is not None else subprocess.run try: diff --git a/src/openultrasast/cpg/queries/config.sc b/src/openultrasast/cpg/queries/config.sc index 28e4aea..e8a8eec 100644 --- a/src/openultrasast/cpg/queries/config.sc +++ b/src/openultrasast/cpg/queries/config.sc @@ -4,17 +4,22 @@ // This query reports each setting call with its arguments split into literals (which the model can evaluate) // and non-literals (which it cannot -- those become `corroborated`, never silently safe). // -// Parameters: cpgFile, settings, function (optional). +// Parameters: cpgFile, settings, function (optional), file (optional: restrict to this source file). +// +// `file` matters more than it looks. A region with no enclosing function -- a module of settings, an +// __init__.py -- used to send function="" and match EVERY setting in the repository, and the driver then +// attributed the whole answer to whichever region asked. One `host='0.0.0.0'` in app.py became eight +// identical entailed findings in eight files that do not contain it. // Output: fenced JSON array of {setting, line, method, args, literalArgs}. // BATCHED, like taint.sc. Single-request form kept for the committed measurements. -@main def exec(cpgFile: String, settings: String = "", function: String = "", requests: String = "") = { +@main def exec(cpgFile: String, settings: String = "", function: String = "", file: String = "", requests: String = "") = { importCpg(cpgFile) def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList - def rowsFor(settingsS: String, functionS: String): List[ujson.Obj] = { + def rowsFor(settingsS: String, functionS: String, fileS: String): List[ujson.Obj] = { val names = split(settingsS) val function = functionS @@ -22,7 +27,8 @@ val calls = { val all = cpg.call.filter(c => matches(c.name)).l - if (function.isEmpty) all else all.filter(_.method.name == function) + val inFile = if (fileS.isEmpty) all else all.filter(_.method.filename.endsWith(fileS)) + if (function.isEmpty) inFile else inFile.filter(_.method.name == function) } val rows = calls.map { call => @@ -51,11 +57,11 @@ val parsed = ujson.read(requests).obj val answers = parsed.map { case (id, req) => def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") - id -> ujson.Arr(rowsFor(field("settings"), field("function")): _*) + id -> ujson.Arr(rowsFor(field("settings"), field("function"), field("file")): _*) } println(ujson.write(ujson.Obj.from(answers))) } else { - println(ujson.write(ujson.Arr(rowsFor(settings, function): _*))) + println(ujson.write(ujson.Arr(rowsFor(settings, function, file): _*))) } println("---OUSAST-CPG-END---") } diff --git a/src/openultrasast/cpg/queries/taint.sc b/src/openultrasast/cpg/queries/taint.sc index b95a7e7..215d1ec 100644 --- a/src/openultrasast/cpg/queries/taint.sc +++ b/src/openultrasast/cpg/queries/taint.sc @@ -17,7 +17,13 @@ // Parameters (comma-separated where plural): // cpgFile, sources, sinks, sanitizers (optional), function (optional: restrict to this method AND any // closure lexically nested inside it -- see `nestedInLabeled`), -// parameterSources ("true" to treat the labeled function's parameters as untrusted -- see below) +// parameterSources ("true" to treat the labeled function's parameters as untrusted -- see below), +// file (optional: restrict sinks to this source file) +// +// `file` is what makes a region with no enclosing function askable at all. Without it such a region sends +// function="" and matches every sink in the repository, and the driver attributes that whole answer to +// whichever region asked. Scoping by file also stops a named function matching a same-named function in +// another module, which a corpus of one-file excerpts could never surface. // // Output: a fenced JSON array of {sink, sinkLine, sinkMethod, source, sanitized, length}. The fence exists // because Joern prints a banner, pass logs and a prompt around whatever a script emits. @@ -36,6 +42,7 @@ sinks: String = "", sanitizers: String = "", function: String = "", + file: String = "", parameterSources: String = "false", requests: String = "" ) = { @@ -43,7 +50,7 @@ def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList - def rowsFor(sourcesS: String, sinksS: String, sanitizersS: String, functionS: String, paramSrc: String): List[ujson.Obj] = { + def rowsFor(sourcesS: String, sinksS: String, sanitizersS: String, functionS: String, paramSrc: String, fileS: String): List[ujson.Obj] = { // Word-boundary matching, never substring. `resolveUrl` contains `resolve`, so a bare-substring sanitizer // test marked the VULNERABLE flow sanitized and the verdict fell back to a captured `res` that names no @@ -104,7 +111,8 @@ val all = cpg.call.filter(c => sinkNames.exists(n => c.name == n || c.code.startsWith(n + "(") || c.code.startsWith(n) || c.methodFullName.contains(n)) ) - if (function.isEmpty) all else all.filter(c => nestedInLabeled(c.method)) + val inFile = if (fileS.isEmpty) all else all.filter(_.method.filename.endsWith(fileS)) + if (function.isEmpty) inFile else inFile.filter(c => nestedInLabeled(c.method)) } val rows = sinkCalls.l.flatMap { sink => @@ -148,11 +156,11 @@ val answers = parsed.map { case (id, req) => def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") val paramSrc = req.obj.get("parameterSources").map(_.str).getOrElse("false") - id -> ujson.Arr(rowsFor(field("sources"), field("sinks"), field("sanitizers"), field("function"), paramSrc): _*) + id -> ujson.Arr(rowsFor(field("sources"), field("sinks"), field("sanitizers"), field("function"), paramSrc, field("file")): _*) } println(ujson.write(ujson.Obj.from(answers))) } else { - println(ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, function, parameterSources): _*))) + println(ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, function, parameterSources, file): _*))) } println("---OUSAST-CPG-END---") } diff --git a/src/openultrasast/model/config_value.py b/src/openultrasast/model/config_value.py index d2f080c..e255c8b 100644 --- a/src/openultrasast/model/config_value.py +++ b/src/openultrasast/model/config_value.py @@ -19,14 +19,20 @@ from .ladder import Rung, Verdict from .specs import ConfigSpec -def request_params(spec: ConfigSpec, *, function: str = "") -> dict[str, object]: - """The query parameters this arbiter sends. Shared with the batcher; see the note in `taint.request_params`.""" - return {"settings": spec.settings, "function": function} +def request_params(spec: ConfigSpec, *, function: str = "", file: str = "") -> dict[str, object]: + """The query parameters this arbiter sends. Shared with the batcher; see the note in `taint.request_params`. + + ``file`` scopes the answer to one source file. A region with no enclosing function sends ``function=""``, + which without this matches the whole repository and gets that answer attributed to it -- one permissive + literal in `app.py` became eight identical entailed findings in eight files that do not contain it. Empty + means unscoped, which is what the single-region pair path still uses. + """ + return {"settings": spec.settings, "function": function, "file": file} -def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "") -> Verdict | None: +def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str = "") -> Verdict | None: """The verdict for a security setting in ``function``, or ``None`` when nothing is established.""" - rows = cpg.run("config", request_params(spec, function=function)) + rows = cpg.run("config", request_params(spec, function=function, file=file)) settings = _settings(rows, spec, function=function) if not settings: return None diff --git a/src/openultrasast/model/pipeline.py b/src/openultrasast/model/pipeline.py index 8c462c3..528f1e9 100644 --- a/src/openultrasast/model/pipeline.py +++ b/src/openultrasast/model/pipeline.py @@ -22,6 +22,7 @@ from __future__ import annotations import json import logging +import re from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any @@ -64,7 +65,9 @@ class ModelFinding: contradiction: str = "" -def _arbitrate(cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *, function: str, parameter_sources: bool) -> Verdict | None: +def _arbitrate( + cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *, path: str, function: str, parameter_sources: bool +) -> Verdict | None: """Route a family to the arbiter that can actually decide it. Dispatching on the spec type rather than on a family name keeps this honest: a family without an arbiter @@ -75,14 +78,15 @@ def _arbitrate(cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *, if isinstance(spec, DominanceSpec): return dominance_verdict(cpg, spec, function=function) if isinstance(spec, ConfigSpec): - return config_verdict(cpg, spec, function=function) - return taint_verdict(cpg, spec, function=function, parameter_sources=parameter_sources) + return config_verdict(cpg, spec, function=function, file=path) + return taint_verdict(cpg, spec, function=function, file=path, parameter_sources=parameter_sources) def scan_region( cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *, + path: str = "", function: str = "", client: Any | None = None, model: str = "", @@ -90,11 +94,11 @@ def scan_region( parameter_sources: bool = False, ) -> list[ModelFinding]: """Findings for one region and family, ordered strongest first.""" - answer = _arbitrate(cpg, spec, function=function, parameter_sources=parameter_sources) + answer = _arbitrate(cpg, spec, path=path, function=function, parameter_sources=parameter_sources) if answer is not None and answer.rung is Rung.ENTAILED: # The graph decided. No model call, and no candidate needs asking about: this region has its finding. - return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.ENTAILED, witness=answer.witness)] + return [ModelFinding(site=_site(path, function, answer.witness), family=spec.family, rung=Rung.ENTAILED, witness=answer.witness)] if answer is not None: # A real flow through a sanitizer. Whether it suffices is the residual, and the model HAS coverage @@ -103,7 +107,9 @@ def scan_region( return [] said = _ask(client, model, residual_question(spec, answer.witness, function)) if said: - return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.CORROBORATED, witness=answer.witness)] + return [ + ModelFinding(site=_site(path, function, answer.witness), family=spec.family, rung=Rung.CORROBORATED, witness=answer.witness) + ] return [] # The model resolved nothing. It is SILENT, not negative — so the enumerator's candidates are put to the @@ -194,9 +200,22 @@ def _ask(client: Any, model: str, question: str) -> bool | None: return bool(payload["vulnerable"]) if isinstance(payload, Mapping) and isinstance(payload.get("vulnerable"), bool) else None -def _site(function: str, witness: str) -> str: - line = witness.partition("(line ")[2].split(",")[0].strip(") ") if "(line " in witness else "?" - return f"{function or '?'}:{line}" +_LINE_IN_WITNESS = re.compile(r"\(line (\d+)") + + +def _site(path: str, function: str, witness: str) -> str: + """``path:line:function`` -- the same shape a candidate id carries, because one consumer parses both. + + An arbitrated finding used to be ``function:line``, which the report layer then read as ``path:line``: + every entailed finding arrived with a function name where its file belonged and no line at all. A + contributor cannot open that, and nothing downstream can match it against a known location. + """ + # A regex, because the witness is prose and the delimiter after the number varies: "(line 17," + # in one family and "(line 17):" in another. Splitting on a comma produced sites like + # `config.py:17): permissive literal '0.0.0.0':?`. + found = _LINE_IN_WITNESS.search(witness) + line = found.group(1) if found else "?" + return f"{path or '?'}:{line}:{function or '?'}" __all__ = ["CANDIDATE_QUESTION", "MAX_JUDGED_CANDIDATES", "ModelFinding", "residual_question", "scan_region"] diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 55ed23d..b54353e 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -6,10 +6,11 @@ two costs that were free at pair scale dominate here: * **CPG construction.** A build takes 5-30 seconds on a forty-line excerpt. One per region would make a repository scan unusable, so this builds **one** and reuses it across every region. * **Model calls.** The enumerator caps candidates at eight per region, which says nothing useful about a scan - with two thousand regions. The budget is a **total for the run**, spent on the highest-ranked regions first, - and whatever it does not reach is counted into ``regions_unjudged`` rather than quietly dropped. A tool that - silently analyses a tenth of a repository and reports a clean result is worse than one that says what it - did not look at. + with two thousand regions. The budget is a **total for the run**, spent on the highest-ranked regions first. + It bounds model calls and nothing else: when it runs out the JUDGE is withheld and the graph keeps deciding, + because arbitration costs nothing. Those regions are counted into ``regions_unasked``, and anything + ``max_regions`` cut before arbitration into ``regions_unjudged``. A tool that silently analyses a tenth of a + repository and reports a clean result is worse than one that says what it did not look at. Ordering puts rung before rank: what the model established outranks what the LLM merely proposed, whatever the region's risk score said beforehand. @@ -58,11 +59,24 @@ class ModelScanResult: by_rung: Mapping[str, int] = field(default_factory=dict) regions_scanned: int = 0 regions_unjudged: int = 0 + # Arbitrated by the graph, but with the judge withheld because the budget was spent. Not the same as + # unjudged, and not the same as clean. + regions_unasked: int = 0 model_calls: int = 0 cost_usd: float = 0.0 seconds: float = 0.0 + # The split, not just the total. Which stage a scan's wall-clock actually goes to decides real questions + # -- whether a warm Joern server would buy anything, whether the budget or the CPG is the ceiling -- and + # a single figure cannot answer any of them. + build_seconds: float = 0.0 + query_seconds: float = 0.0 degradations: tuple[Mapping[str, object], ...] = () + @property + def arbitrate_seconds(self) -> float: + """What is left once the CPG is built and queried: the judge calls and the arbiters themselves.""" + return round(max(self.seconds - self.build_seconds - self.query_seconds, 0.0), 2) + def scan_repository( root: Path, @@ -79,12 +93,15 @@ def scan_repository( degradations: list[Mapping[str, object]] = [] taxonomy = load_families() + build_started = time.monotonic() cpg = backend.build(root) + build_seconds = round(time.monotonic() - build_started, 2) if cpg is None: return ModelScanResult( by_rung=_empty_tally(), regions_unjudged=len(regions), seconds=round(time.monotonic() - started, 2), + build_seconds=build_seconds, degradations=({"stage": "model", "reason": "cpg_build_failed"},), ) @@ -111,6 +128,7 @@ def scan_repository( # per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while # the queries themselves are milliseconds once the CPG is loaded. rows_by_id: dict[str, list[object]] = {} + query_started = time.monotonic() for kind, requests in _grouped(work).items(): batch = getattr(cpg, "run_batch", None) if callable(batch): @@ -119,17 +137,23 @@ def scan_repository( for rid, params in requests.items(): answered = cpg.run(kind, params) rows_by_id[rid] = list(answered) if isinstance(answered, list) else [] + query_seconds = round(time.monotonic() - query_started, 2) # Phase 3: arbitrate from the rows already in hand. The arbiters are unchanged: each is handed a # CpgResult that simply returns its own prefetched rows. # - # `judged` counts regions actually ARBITRATED, not regions collected: a region the budget stopped before - # is unjudged, and reporting it as scanned is the silent truncation this exists to prevent. + # The budget bounds MODEL CALLS, and nothing else. Arbitration is the CPG alone -- it costs no call, no + # token and no money -- so an exhausted budget withholds the JUDGE and lets the graph keep deciding. It + # used to break the loop instead, which spent a paid limit to stop unpaid work: on VAmPI that left 16 of + # 22 regions unexamined and missed a BOLA the graph entails for free. + # + # `judged` counts regions actually ARBITRATED and `unasked` those arbitrated with no judge behind them. + # Both are reported: a region the graph settled and a region the graph was silent about but nobody could + # afford to ask are not the same result, and collapsing them is the silent truncation this prevents. judged: set[tuple[str, str | None]] = set() + unasked: set[tuple[str, str | None]] = set() for rid, region, spec in work: - if counted is not None and counted.calls >= limits.max_model_calls: - degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unjudged": max(len(regions) - len(judged), 0)}) - break + exhausted = counted is not None and counted.calls >= limits.max_model_calls family = getattr(spec, "family", "") prefetched = CpgResult(cpg_path=cpg.cpg_path, run=_prefetched(rows_by_id.get(rid, []))) try: @@ -138,8 +162,9 @@ def scan_repository( found = scan_region( prefetched, spec, + path=region.path, function=region.function or "", - client=counted, + client=None if exhausted else counted, model=model, candidates=candidates, parameter_sources=False, # a repository's parameters are not all attacker input @@ -149,8 +174,13 @@ def scan_repository( degradations.append({"stage": "model", "reason": "region_failed", "path": region.path}) continue judged.add((region.path, region.function)) + if exhausted: + unasked.add((region.path, region.function)) collected.extend((finding, region.rank) for finding in found) + if unasked: + degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unasked": len(unasked)}) + scanned = len(judged) findings = _ordered(collected) @@ -159,9 +189,12 @@ def scan_repository( by_rung=_tally(findings), regions_scanned=scanned, regions_unjudged=max(len(regions) - scanned, 0), + regions_unasked=len(unasked), model_calls=counted.calls if counted is not None else 0, cost_usd=round(_cost(counted), 4), seconds=round(time.monotonic() - started, 2), + build_seconds=build_seconds, + query_seconds=query_seconds, degradations=tuple(degradations), ) @@ -188,9 +221,9 @@ def _grouped(work: Sequence[tuple[str, ScanRegion, ArbiterSpec]]) -> dict[str, d if isinstance(spec, DominanceSpec): kind, params = "dominance", dominance_params(spec, function=function) elif isinstance(spec, ConfigSpec): - kind, params = "config", config_params(spec, function=function) + kind, params = "config", config_params(spec, function=function, file=region.path) else: - kind, params = "taint", taint_params(spec, function=function, parameter_sources=False) + kind, params = "taint", taint_params(spec, function=function, file=region.path, parameter_sources=False) grouped.setdefault(kind, {})[rid] = params return grouped diff --git a/src/openultrasast/model/taint.py b/src/openultrasast/model/taint.py index ce17457..a251eee 100644 --- a/src/openultrasast/model/taint.py +++ b/src/openultrasast/model/taint.py @@ -19,29 +19,35 @@ from .ladder import Rung, Verdict from .specs import TaintSpec -def request_params(spec: TaintSpec, *, function: str = "", parameter_sources: bool = False) -> dict[str, object]: +def request_params(spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False) -> dict[str, object]: """The query parameters this arbiter sends. Shared with the batcher so there is one definition, not two. Two copies of a parameter dict is precisely the duplication that produced five bugs of one shape in the predecessor: a change made in one place and not its sibling. + + ``file`` scopes the answer to one source file. A region with no enclosing function sends ``function=""``, + which without this matches the whole repository and gets that answer attributed to it -- one permissive + literal in `app.py` became eight identical entailed findings in eight files that do not contain it. Empty + means unscoped, which is what the single-region pair path still uses. """ return { "sources": spec.sources, "sinks": spec.sinks, "sanitizers": spec.sanitizers, "function": function, + "file": file, "parameterSources": "true" if parameter_sources else "false", } -def verdict(cpg: CpgResult, spec: TaintSpec, *, function: str = "", parameter_sources: bool = False) -> Verdict | None: +def verdict(cpg: CpgResult, spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False) -> Verdict | None: """The strongest verdict the taint query supports for ``spec`` in ``function``, or ``None`` to stay at suspicion. ``parameter_sources`` additionally treats the labeled function's own parameters as untrusted. That is right for a function-level pair, where the function boundary *is* the trust boundary, and wrong for a whole repository, where most parameters carry internal values — so it is off by default and the caller opts in. """ - rows = cpg.run("taint", request_params(spec, function=function, parameter_sources=parameter_sources)) + rows = cpg.run("taint", request_params(spec, function=function, file=file, parameter_sources=parameter_sources)) flows = _flows(rows, function=function) # A sink whose *shape* is safe is the fix, not the bug. `execute(sql, params)` binds rather than # interpolates and `printf("literal", x)` has a constant format string, so the flow that reaches them is diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index 84f7409..2a39831 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -162,3 +162,97 @@ def test_a_batch_result_that_omits_a_request_returns_nothing_for_it() -> None: backend = JoernBackend(runner=lambda c, **k: _Partial()) rows = backend.query_batch(Path("/tmp/c.bin"), "taint", {"r1": {}, "r2": {}}) assert rows.get("r1") == [] and "r2" not in rows + + +def test_a_built_cpg_offers_the_batch_path_to_its_driver(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The capability must be reachable through the object the driver is handed, not merely present. + + ``query_batch`` was implemented, tested and committed while ``build`` returned a result carrying only + ``run``. The driver discovers batching by looking for ``run_batch`` on that result, found nothing, and + fell back to one JVM per region -- silently, because every test called ``query_batch`` directly. A + repository scan is what made it visible: ~14s per region-family, one ``--param function=`` invocation at + a time. + """ + from openultrasast.cpg.backend import BEGIN, END, JoernBackend + + scripts: list[list[str]] = [] + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + class _Done: + returncode = 0 + stderr = "" + stdout = f'{BEGIN}\n{{"r1": [{{"sink": "os.system(x)"}}]}}\n{END}\n' + + if "--output" in command: # the parse step; make the artifact it promises + Path(command[command.index("--output") + 1]).write_text("cpg") + else: + scripts.append(list(command)) + return _Done() + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + result = JoernBackend(runner=runner).build(tmp_path) + + assert result is not None + assert result.run_batch is not None, "a backend that can batch must expose it, or the driver cannot find it" + rows = result.run_batch("taint", {"r1": {"sources": ("request.args",), "sinks": ("os.system",)}}) + assert rows["r1"], "the batch path must reach the engine, not just exist" + assert len(scripts) == 1, "one invocation for the batch" + assert not any(arg.startswith("function=") for arg in scripts[0]), "a batch carries requests, not one function" + + +def test_the_engine_runs_under_a_bounded_heap(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A JVM with no -Xmx claims a quarter of physical RAM. + + That is not a tuning preference. The first repository measurements were killed under memory pressure at + different points, so their numbers described the host rather than the engine, and on a contributor's + laptop an unbounded heap is the difference between a scan running in the background and a scan the + machine notices. + """ + from openultrasast.cpg.backend import JoernBackend + + commands: list[list[str]] = [] + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + class _Done: + returncode = 0 + stderr = "" + stdout = "" + + commands.append(list(command)) + if "--output" in command: + Path(command[command.index("--output") + 1]).write_text("cpg") + return _Done() + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + monkeypatch.delenv("OPENULTRASAST_CPG_HEAP_MB", raising=False) + result = JoernBackend(runner=runner).build(tmp_path) + assert result is not None + result.run("taint", {}) + + assert commands, "nothing ran" + for command in commands: + assert any(arg.startswith("-J-Xmx") for arg in command), f"unbounded heap in {command[0]}" + + +def test_the_operator_can_raise_the_heap(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A bound that cannot be lifted is a bound that gets removed. A big repository may need a big heap.""" + from openultrasast.cpg.backend import JoernBackend + + commands: list[list[str]] = [] + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + class _Done: + returncode = 0 + stderr = "" + stdout = "" + + commands.append(list(command)) + if "--output" in command: + Path(command[command.index("--output") + 1]).write_text("cpg") + return _Done() + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + monkeypatch.setenv("OPENULTRASAST_CPG_HEAP_MB", "6144") + JoernBackend(runner=runner).build(tmp_path) + + assert "-J-Xmx6144m" in commands[0] diff --git a/tests/test_model_pipeline.py b/tests/test_model_pipeline.py index 6711763..74c5f63 100644 --- a/tests/test_model_pipeline.py +++ b/tests/test_model_pipeline.py @@ -79,6 +79,22 @@ def test_an_entailed_site_becomes_a_finding_with_no_model_call() -> None: assert client.calls == 0 +def test_an_arbitrated_finding_names_its_file_the_way_a_candidate_does() -> None: + """One consumer parses both shapes, so both must be ``path:line:function``. + + An arbitrated finding was ``function:line``, which the report layer read as ``path:line``: every entailed + finding reached a contributor with a function name where its file belonged and no line at all. A whole + repository scan is what made it visible -- `model:access_control:get_all_books:?`, path `get_all_books`. + """ + from openultrasast.model.pipeline import scan_region + + findings = scan_region(_cpg(ENTAILING), _spec(), path="models/user_model.py", function="get_user", client=_Client(), model="m") + + path, _, rest = findings[0].site.partition(":") + assert path == "models/user_model.py", "the file, not the function" + assert rest.endswith(":get_user") + + def test_a_sanitized_flow_the_judge_confirms_becomes_corroborated() -> None: from openultrasast.model.ladder import Rung from openultrasast.model.pipeline import scan_region @@ -244,3 +260,16 @@ def test_the_dominance_question_describes_the_evidence_the_model_actually_has() assert "siblings have" not in q, "the corroborated case has no guarded siblings to appeal to" assert "sibling handlers are guarded" not in q assert "guard" in q.lower() and "public" in q.lower(), "it must still ask whether the operation needs a guard" + + +def test_a_witness_line_is_read_whatever_follows_the_number() -> None: + """The delimiter after the line number differs by family: `(line 17,` in one, `(line 17):` in another. + + Splitting on the comma produced sites like `config.py:17): permissive literal '0.0.0.0':?` -- a line + number no editor can open and no consumer can compare. + """ + from openultrasast.model.pipeline import _site + + assert _site("config.py", "", "permissive literal '0.0.0.0' (line 17): permissive") == "config.py:17:?" + assert _site("a.py", "run", "flow reaches os.system (line 9, via x)") == "a.py:9:run" + assert _site("a.py", "run", "no line at all") == "a.py:?:run" diff --git a/tests/test_model_scan.py b/tests/test_model_scan.py index aff2e87..a384c71 100644 --- a/tests/test_model_scan.py +++ b/tests/test_model_scan.py @@ -57,8 +57,11 @@ def test_a_failed_build_yields_an_empty_result_not_an_exception() -> None: assert any("cpg_build_failed" in str(d) for d in result.degradations) -def test_the_budget_is_a_total_for_the_run_and_the_remainder_is_counted(tmp_path) -> None: - """Req 3.2, 3.3: silently truncating is the failure this exists to prevent. +def test_an_exhausted_budget_withholds_the_judge_and_not_the_graph(tmp_path) -> None: + """Req 3.2, 3.3: the budget bounds MODEL CALLS. Arbitration is free and must keep running. + + The driver used to break the region loop when the budget ran out, spending a paid limit to stop unpaid + work. On VAmPI that left 16 of 22 regions unexamined and missed a BOLA the graph entails for nothing. Real files on disk, because the driver enumerates candidates from them. A region pointing at a path that does not exist yields no candidates, no model calls, and a budget that never trips -- which would make @@ -86,11 +89,39 @@ def test_the_budget_is_a_total_for_the_run_and_the_remainder_is_counted(tmp_path budget=ScanBudget(max_model_calls=5, max_regions=50), ) assert result.model_calls > 0, "the driver must enumerate candidates and ask the judge about them" - assert result.regions_unjudged > 0 - assert result.regions_scanned + result.regions_unjudged == 50 + assert result.regions_scanned == 50, "every region is still arbitrated; only the judge is withheld" + assert result.regions_unjudged == 0 + assert result.regions_unasked > 0, "the regions the budget could not afford a judge for are counted" assert any("budget_exhausted" in str(d) for d in result.degradations) +def test_a_region_the_budget_could_not_afford_is_not_reported_as_clean(tmp_path) -> None: + """`unasked` is the honesty counter: silent-because-decided and silent-because-unaffordable differ.""" + from openultrasast.model.scan import ScanBudget, scan_repository + + class _Client: + def complete(self, **kw): # type: ignore[no-untyped-def] + from openultrasast.tool_hunter import ChatResponse + + return ChatResponse(content='{"vulnerable": true}') + + source = "import os\n\n\ndef run(cmd):\n os.system(cmd)\n" + for i in range(10): + (tmp_path / f"f{i}.py").write_text(source) + regions = tuple(_region(path=f"f{i}.py") for i in range(10)) + + generous = scan_repository( + tmp_path, regions, backend=_Backend(), client=_Client(), model="m", budget=ScanBudget(max_model_calls=500, max_regions=10) + ) + starved = scan_repository( + tmp_path, regions, backend=_Backend(), client=_Client(), model="m", budget=ScanBudget(max_model_calls=1, max_regions=10) + ) + + assert generous.regions_unasked == 0, "nothing went unasked when the budget covered the run" + assert starved.regions_unasked > generous.regions_unasked + assert starved.regions_scanned == generous.regions_scanned == 10 + + def test_the_budget_is_spent_on_the_highest_ranked_regions_first() -> None: from openultrasast.model.scan import ScanBudget, scan_repository @@ -186,3 +217,60 @@ def test_the_driver_issues_one_query_invocation_per_kind_not_one_per_region(tmp_ assert not any(i.startswith("single:") for i in invocations), f"unbatched calls remain: {invocations[:4]}" # 30 regions x 2 families would have been 60 invocations; batching makes it one per query kind. assert len(invocations) <= 3, f"expected one invocation per query kind, got {len(invocations)}" + + +class _SlowBackend: + """A backend whose build and whose queries each cost a measurable, distinguishable amount of time.""" + + def __init__(self, build_cost: float, query_cost: float) -> None: + self.build_cost = build_cost + self.query_cost = query_cost + + def available(self) -> bool: + return True + + def build(self, root): # type: ignore[no-untyped-def] + import time + + from openultrasast.cpg.backend import CpgResult + + time.sleep(self.build_cost) + return CpgResult(cpg_path=Path("cpg.bin"), run=self._run) + + def _run(self, query, params): # type: ignore[no-untyped-def] + import time + + time.sleep(self.query_cost) + return [] + + +def test_the_scan_reports_where_its_wall_clock_went() -> None: + """contributor-scan 2.2/2.3: build against query is a decision input, and a total cannot answer it. + + Whether a warm Joern server would buy anything depends on the ratio between a CPG build and the queries + over it. Reporting one figure for the scan makes that question unanswerable from an artifact. + """ + from openultrasast.model.scan import scan_repository + + result = scan_repository( + Path("/repo"), + (_region(),), + backend=_SlowBackend(build_cost=0.30, query_cost=0.10), + client=None, + model="", + ) + + assert result.build_seconds >= 0.29, "the build was not timed" + assert result.query_seconds >= 0.09, "the queries were not timed" + assert result.seconds >= result.build_seconds + result.query_seconds + assert result.arbitrate_seconds == round(max(result.seconds - result.build_seconds - result.query_seconds, 0.0), 2) + + +def test_a_failed_build_still_reports_what_the_build_cost() -> None: + """A build that fails after two minutes is the most important timing in the artifact, not the least.""" + from openultrasast.model.scan import scan_repository + + result = scan_repository(Path("/repo"), (_region(),), backend=_Backend(fail=True), client=None, model="") + + assert result.build_seconds >= 0.0 + assert result.query_seconds == 0.0, "nothing was queried" diff --git a/tests/test_model_taint.py b/tests/test_model_taint.py index ee8a5e4..5dac79e 100644 --- a/tests/test_model_taint.py +++ b/tests/test_model_taint.py @@ -395,3 +395,23 @@ def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() ] answer = verdict(_cpg(rows), _spec(), function="run") assert answer is not None and answer.rung is Rung.ENTAILED + + +def test_a_region_scopes_its_question_to_its_own_file() -> None: + """A region with no enclosing function must not be answered with the whole repository. + + Sending `function=""` unscoped matched every hit in the tree and the driver attributed all of it to + whichever region asked: one permissive literal in app.py became eight identical entailed findings in + eight files that do not contain it. + """ + from openultrasast.model.config_value import request_params as config_params + from openultrasast.model.specs import config_specs, taint_specs + from openultrasast.model.taint import request_params as taint_params + + taint = taint_params(taint_specs(language="python")["injection"], function="", file="models/user_model.py") + config = config_params(config_specs(language="python")["config_secrets"], function="", file="config.py") + + assert taint["file"] == "models/user_model.py" + assert config["file"] == "config.py" + # The pair path stays unscoped, so the committed pair measurements remain reproducible. + assert taint_params(taint_specs(language="python")["injection"], function="run")["file"] == ""