mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 20:19:29 +00:00
feat(model): first repository measurement, and the four things it found
contributor-scan 2.2. VAmPI at a pinned commit, scanned whole rather than as excerpts, with benchmarks/repos/measure.py recording build time, query time, arbitration time, peak memory, cost, unjudged regions, and whether each known vulnerability was found and at which rung. The corpus could not have found any of this. Every defect below needs a repository to become visible, and three of them were invisible behind a passing test suite. **The batch path was unreachable.** query_batch was implemented, tested and committed while CpgResult exposed only `run`, so the driver's capability check found nothing and fell back to one JVM per region-family: the first attempt at this measurement ran eighteen minutes on 520 lines, issuing `--param function=` one at a time. Every test called query_batch directly, so none of them noticed. CpgResult now carries run_batch and a test asserts the capability is reachable through the object the driver is handed. The earlier 275s->106s improvement attributed to batching came from region deduplication alone. **The budget stopped free work.** It bounds model calls, but was enforced by breaking the region loop -- spending a paid limit to stop unpaid CPG arbitration. 6 of 22 regions were examined and VAmPI's BOLA, which the graph entails for nothing, was never reached. It now withholds the judge only. 22 of 22 regions arbitrated, 17 counted as regions_unasked (silent because unaffordable is not silent because decided), and the BOLA is entailed by the model layer. **A region with no function asked about the whole repository.** Sending function="" matched every hit in the tree and the driver attributed that answer to whoever asked: one `host='0.0.0.0'` in app.py became eight identical entailed findings in eight files that do not contain it. Queries now carry the region's file. Entailed findings 14 -> 7, query time 70.9s -> 40.9s. The single-request form stays unscoped so the committed pair measurements remain reproducible. **The JVM had no heap bound.** Joern claimed a quarter of physical RAM and runs were killed under memory pressure at different points, so the numbers described the host rather than the engine. -J-Xmx2048m by default, OPENULTRASAST_CPG_HEAP_MB to raise it. Also: arbitrated findings carried `function:line` where the report layer read `path:line`, so every entailed finding reached a contributor with a function name where its file belonged; and the witness line was parsed by splitting on a comma, which produced sites like `config.py:17): permissive literal '0.0.0.0':?`. Reading: GO on one CPG per repository -- 6.2s to build, 177s end to end, 869MB, $0.0067, nothing infeasible. NO-GO on region-scoped taint as the only shape: VAmPI's SQL injection runs from a handler parameter into a function in another module, and a per-function region sees neither end. That is the ordinary shape of a real injection bug, so it is recorded as task 2.4 along with three smaller defects this run exposed. Gates byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
8c387306be
commit
177c7f3719
@@ -883,10 +883,14 @@ def _model_payload(result: object, *, unjudged_paths: tuple[str, ...] = ()) -> d
|
||||
"by_rung": dict(getattr(result, "by_rung", {}) or {}),
|
||||
"regions_scanned": getattr(result, "regions_scanned", 0),
|
||||
"regions_unjudged": getattr(result, "regions_unjudged", 0),
|
||||
"regions_unasked": getattr(result, "regions_unasked", 0),
|
||||
"unjudged_sample": list(unjudged_paths),
|
||||
"model_calls": getattr(result, "model_calls", 0),
|
||||
"cost_usd": getattr(result, "cost_usd", 0.0),
|
||||
"seconds": getattr(result, "seconds", 0.0),
|
||||
"build_seconds": getattr(result, "build_seconds", 0.0),
|
||||
"query_seconds": getattr(result, "query_seconds", 0.0),
|
||||
"arbitrate_seconds": getattr(result, "arbitrate_seconds", 0.0),
|
||||
}
|
||||
|
||||
|
||||
@@ -952,7 +956,7 @@ def _finding_from_model(item: object) -> StaticFinding:
|
||||
"""A model finding as a StaticFinding, carrying its rung and witness (Req 5.1)."""
|
||||
site = str(getattr(item, "site", ""))
|
||||
path, _, rest = site.partition(":")
|
||||
line = rest.split(":")[0] if rest else ""
|
||||
line, _, function = rest.partition(":")
|
||||
rung = getattr(item, "rung", None)
|
||||
witness = str(getattr(item, "witness", "") or getattr(item, "contradiction", ""))
|
||||
family = str(getattr(item, "family", ""))
|
||||
@@ -965,7 +969,7 @@ def _finding_from_model(item: object) -> StaticFinding:
|
||||
evidence_level="suspicion",
|
||||
rationale=witness or f"the model layer reported a {family} candidate",
|
||||
line=int(line) if line.isdigit() else None,
|
||||
function_name=None,
|
||||
function_name=function or None,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
|
||||
@@ -40,6 +40,12 @@ END = "---OUSAST-CPG-END---"
|
||||
|
||||
BUILD_TIMEOUT_SECONDS = 900 # a large tree takes minutes; the go/no-go records what it actually cost
|
||||
QUERY_TIMEOUT_SECONDS = 300
|
||||
# A JVM with no -Xmx takes a quarter of physical RAM for its heap. On a contributor's laptop that is the
|
||||
# difference between a scan running in the background and a scan the machine notices, and it made the
|
||||
# first repository measurements unreproducible: runs were killed under memory pressure at different
|
||||
# points, so the numbers described the host, not the engine. Bound it, and let the operator raise it.
|
||||
CPG_HEAP_MB = 2048
|
||||
HEAP_ENV = "OPENULTRASAST_CPG_HEAP_MB"
|
||||
|
||||
Runner = Callable[..., subprocess.CompletedProcess[str]]
|
||||
|
||||
@@ -61,10 +67,18 @@ def extract_payload(stdout: str) -> object | None:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CpgResult:
|
||||
"""A built CPG and the way to query it. Opaque: only ``model/`` reads through ``run``."""
|
||||
"""A built CPG and the ways to query it. Opaque: only ``model/`` reads through these.
|
||||
|
||||
``run_batch`` is optional so a backend that cannot answer many questions in one invocation still fits
|
||||
this shape -- the driver falls back to ``run``, once per request. But a backend that CAN batch must
|
||||
attach it HERE, because this is the driver's only way to discover the capability. Leaving it off does
|
||||
not fail: it silently costs a JVM per region, which is how batching sat unreachable behind a passing
|
||||
test suite until a repository scan made the cost visible.
|
||||
"""
|
||||
|
||||
cpg_path: Path
|
||||
run: Callable[[str, Mapping[str, object]], object | None]
|
||||
run_batch: Callable[[str, Mapping[str, Mapping[str, object]]], dict[str, list[object]]] | None = None
|
||||
|
||||
|
||||
class CpgBackend(Protocol):
|
||||
@@ -94,6 +108,7 @@ class JoernBackend:
|
||||
runner: Runner | None = None
|
||||
build_timeout: int = BUILD_TIMEOUT_SECONDS
|
||||
query_timeout: int = QUERY_TIMEOUT_SECONDS
|
||||
heap_mb: int = 0 # 0 means read the environment, then fall back to CPG_HEAP_MB
|
||||
queries_dir: Path = field(default_factory=lambda: QUERIES_DIR)
|
||||
|
||||
def available(self) -> bool:
|
||||
@@ -108,13 +123,17 @@ class JoernBackend:
|
||||
return None
|
||||
scratch = Path(tempfile.mkdtemp(prefix="ousast-cpg-"))
|
||||
cpg_path = scratch / "cpg.bin"
|
||||
command = [parse or "joern-parse", str(root), "--output", str(cpg_path)]
|
||||
command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)]
|
||||
completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
|
||||
if completed is None or completed.returncode != 0 or not cpg_path.is_file():
|
||||
detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout"
|
||||
logger.warning("cpg build failed for %s: %s", root, detail)
|
||||
return None
|
||||
return CpgResult(cpg_path=cpg_path, run=lambda query, params: self.query(cpg_path, query, params))
|
||||
return CpgResult(
|
||||
cpg_path=cpg_path,
|
||||
run=lambda query, params: self.query(cpg_path, query, params),
|
||||
run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests),
|
||||
)
|
||||
|
||||
def query_batch(self, cpg_path: Path, query: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]:
|
||||
"""Run ONE script invocation carrying many requests, keyed back to their ids.
|
||||
@@ -135,6 +154,7 @@ class JoernBackend:
|
||||
payload = json.dumps({rid: {k: _render(v) for k, v in req.items()} for rid, req in requests.items()})
|
||||
command = [
|
||||
shutil.which("joern") or "joern",
|
||||
self._heap_flag(),
|
||||
"--script",
|
||||
str(script),
|
||||
"--param",
|
||||
@@ -159,7 +179,7 @@ class JoernBackend:
|
||||
if not script.is_file():
|
||||
logger.warning("no such cpg query: %s", script)
|
||||
return None
|
||||
command = [shutil.which("joern") or "joern", "--script", str(script), "--param", f"cpgFile={cpg_path}"]
|
||||
command = [shutil.which("joern") or "joern", self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"]
|
||||
for key, value in sorted(params.items()):
|
||||
command += ["--param", f"{key}={_render(value)}"]
|
||||
# Joern writes a `workspace/` beside the working directory; run it inside the CPG's own scratch dir so
|
||||
@@ -171,6 +191,13 @@ class JoernBackend:
|
||||
return None
|
||||
return extract_payload(completed.stdout or "")
|
||||
|
||||
def _heap_flag(self) -> str:
|
||||
"""``-J-Xmx``: both launchers forward ``-J`` arguments to the JVM."""
|
||||
if self.heap_mb > 0:
|
||||
return f"-J-Xmx{self.heap_mb}m"
|
||||
configured = os.environ.get(HEAP_ENV, "").strip()
|
||||
return f"-J-Xmx{configured if configured.isdigit() else CPG_HEAP_MB}m"
|
||||
|
||||
def _run(self, command: list[str], *, timeout: int, cwd: Path | None = None) -> subprocess.CompletedProcess[str] | None:
|
||||
run = self.runner if self.runner is not None else subprocess.run
|
||||
try:
|
||||
|
||||
@@ -4,17 +4,22 @@
|
||||
// This query reports each setting call with its arguments split into literals (which the model can evaluate)
|
||||
// and non-literals (which it cannot -- those become `corroborated`, never silently safe).
|
||||
//
|
||||
// Parameters: cpgFile, settings, function (optional).
|
||||
// Parameters: cpgFile, settings, function (optional), file (optional: restrict to this source file).
|
||||
//
|
||||
// `file` matters more than it looks. A region with no enclosing function -- a module of settings, an
|
||||
// __init__.py -- used to send function="" and match EVERY setting in the repository, and the driver then
|
||||
// attributed the whole answer to whichever region asked. One `host='0.0.0.0'` in app.py became eight
|
||||
// identical entailed findings in eight files that do not contain it.
|
||||
// Output: fenced JSON array of {setting, line, method, args, literalArgs}.
|
||||
|
||||
// BATCHED, like taint.sc. Single-request form kept for the committed measurements.
|
||||
|
||||
@main def exec(cpgFile: String, settings: String = "", function: String = "", requests: String = "") = {
|
||||
@main def exec(cpgFile: String, settings: String = "", function: String = "", file: String = "", requests: String = "") = {
|
||||
importCpg(cpgFile)
|
||||
|
||||
def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList
|
||||
|
||||
def rowsFor(settingsS: String, functionS: String): List[ujson.Obj] = {
|
||||
def rowsFor(settingsS: String, functionS: String, fileS: String): List[ujson.Obj] = {
|
||||
val names = split(settingsS)
|
||||
val function = functionS
|
||||
|
||||
@@ -22,7 +27,8 @@
|
||||
|
||||
val calls = {
|
||||
val all = cpg.call.filter(c => matches(c.name)).l
|
||||
if (function.isEmpty) all else all.filter(_.method.name == function)
|
||||
val inFile = if (fileS.isEmpty) all else all.filter(_.method.filename.endsWith(fileS))
|
||||
if (function.isEmpty) inFile else inFile.filter(_.method.name == function)
|
||||
}
|
||||
|
||||
val rows = calls.map { call =>
|
||||
@@ -51,11 +57,11 @@
|
||||
val parsed = ujson.read(requests).obj
|
||||
val answers = parsed.map { case (id, req) =>
|
||||
def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("")
|
||||
id -> ujson.Arr(rowsFor(field("settings"), field("function")): _*)
|
||||
id -> ujson.Arr(rowsFor(field("settings"), field("function"), field("file")): _*)
|
||||
}
|
||||
println(ujson.write(ujson.Obj.from(answers)))
|
||||
} else {
|
||||
println(ujson.write(ujson.Arr(rowsFor(settings, function): _*)))
|
||||
println(ujson.write(ujson.Arr(rowsFor(settings, function, file): _*)))
|
||||
}
|
||||
println("---OUSAST-CPG-END---")
|
||||
}
|
||||
|
||||
@@ -17,7 +17,13 @@
|
||||
// Parameters (comma-separated where plural):
|
||||
// cpgFile, sources, sinks, sanitizers (optional), function (optional: restrict to this method AND any
|
||||
// closure lexically nested inside it -- see `nestedInLabeled`),
|
||||
// parameterSources ("true" to treat the labeled function's parameters as untrusted -- see below)
|
||||
// parameterSources ("true" to treat the labeled function's parameters as untrusted -- see below),
|
||||
// file (optional: restrict sinks to this source file)
|
||||
//
|
||||
// `file` is what makes a region with no enclosing function askable at all. Without it such a region sends
|
||||
// function="" and matches every sink in the repository, and the driver attributes that whole answer to
|
||||
// whichever region asked. Scoping by file also stops a named function matching a same-named function in
|
||||
// another module, which a corpus of one-file excerpts could never surface.
|
||||
//
|
||||
// Output: a fenced JSON array of {sink, sinkLine, sinkMethod, source, sanitized, length}. The fence exists
|
||||
// because Joern prints a banner, pass logs and a prompt around whatever a script emits.
|
||||
@@ -36,6 +42,7 @@
|
||||
sinks: String = "",
|
||||
sanitizers: String = "",
|
||||
function: String = "",
|
||||
file: String = "",
|
||||
parameterSources: String = "false",
|
||||
requests: String = ""
|
||||
) = {
|
||||
@@ -43,7 +50,7 @@
|
||||
|
||||
def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList
|
||||
|
||||
def rowsFor(sourcesS: String, sinksS: String, sanitizersS: String, functionS: String, paramSrc: String): List[ujson.Obj] = {
|
||||
def rowsFor(sourcesS: String, sinksS: String, sanitizersS: String, functionS: String, paramSrc: String, fileS: String): List[ujson.Obj] = {
|
||||
|
||||
// Word-boundary matching, never substring. `resolveUrl` contains `resolve`, so a bare-substring sanitizer
|
||||
// test marked the VULNERABLE flow sanitized and the verdict fell back to a captured `res` that names no
|
||||
@@ -104,7 +111,8 @@
|
||||
val all = cpg.call.filter(c =>
|
||||
sinkNames.exists(n => c.name == n || c.code.startsWith(n + "(") || c.code.startsWith(n) || c.methodFullName.contains(n))
|
||||
)
|
||||
if (function.isEmpty) all else all.filter(c => nestedInLabeled(c.method))
|
||||
val inFile = if (fileS.isEmpty) all else all.filter(_.method.filename.endsWith(fileS))
|
||||
if (function.isEmpty) inFile else inFile.filter(c => nestedInLabeled(c.method))
|
||||
}
|
||||
|
||||
val rows = sinkCalls.l.flatMap { sink =>
|
||||
@@ -148,11 +156,11 @@
|
||||
val answers = parsed.map { case (id, req) =>
|
||||
def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("")
|
||||
val paramSrc = req.obj.get("parameterSources").map(_.str).getOrElse("false")
|
||||
id -> ujson.Arr(rowsFor(field("sources"), field("sinks"), field("sanitizers"), field("function"), paramSrc): _*)
|
||||
id -> ujson.Arr(rowsFor(field("sources"), field("sinks"), field("sanitizers"), field("function"), paramSrc, field("file")): _*)
|
||||
}
|
||||
println(ujson.write(ujson.Obj.from(answers)))
|
||||
} else {
|
||||
println(ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, function, parameterSources): _*)))
|
||||
println(ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, function, parameterSources, file): _*)))
|
||||
}
|
||||
println("---OUSAST-CPG-END---")
|
||||
}
|
||||
|
||||
@@ -19,14 +19,20 @@ from .ladder import Rung, Verdict
|
||||
from .specs import ConfigSpec
|
||||
|
||||
|
||||
def request_params(spec: ConfigSpec, *, function: str = "") -> dict[str, object]:
|
||||
"""The query parameters this arbiter sends. Shared with the batcher; see the note in `taint.request_params`."""
|
||||
return {"settings": spec.settings, "function": function}
|
||||
def request_params(spec: ConfigSpec, *, function: str = "", file: str = "") -> dict[str, object]:
|
||||
"""The query parameters this arbiter sends. Shared with the batcher; see the note in `taint.request_params`.
|
||||
|
||||
``file`` scopes the answer to one source file. A region with no enclosing function sends ``function=""``,
|
||||
which without this matches the whole repository and gets that answer attributed to it -- one permissive
|
||||
literal in `app.py` became eight identical entailed findings in eight files that do not contain it. Empty
|
||||
means unscoped, which is what the single-region pair path still uses.
|
||||
"""
|
||||
return {"settings": spec.settings, "function": function, "file": file}
|
||||
|
||||
|
||||
def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "") -> Verdict | None:
|
||||
def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str = "") -> Verdict | None:
|
||||
"""The verdict for a security setting in ``function``, or ``None`` when nothing is established."""
|
||||
rows = cpg.run("config", request_params(spec, function=function))
|
||||
rows = cpg.run("config", request_params(spec, function=function, file=file))
|
||||
settings = _settings(rows, spec, function=function)
|
||||
if not settings:
|
||||
return None
|
||||
|
||||
@@ -22,6 +22,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
@@ -64,7 +65,9 @@ class ModelFinding:
|
||||
contradiction: str = ""
|
||||
|
||||
|
||||
def _arbitrate(cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *, function: str, parameter_sources: bool) -> Verdict | None:
|
||||
def _arbitrate(
|
||||
cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *, path: str, function: str, parameter_sources: bool
|
||||
) -> Verdict | None:
|
||||
"""Route a family to the arbiter that can actually decide it.
|
||||
|
||||
Dispatching on the spec type rather than on a family name keeps this honest: a family without an arbiter
|
||||
@@ -75,14 +78,15 @@ def _arbitrate(cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, *,
|
||||
if isinstance(spec, DominanceSpec):
|
||||
return dominance_verdict(cpg, spec, function=function)
|
||||
if isinstance(spec, ConfigSpec):
|
||||
return config_verdict(cpg, spec, function=function)
|
||||
return taint_verdict(cpg, spec, function=function, parameter_sources=parameter_sources)
|
||||
return config_verdict(cpg, spec, function=function, file=path)
|
||||
return taint_verdict(cpg, spec, function=function, file=path, parameter_sources=parameter_sources)
|
||||
|
||||
|
||||
def scan_region(
|
||||
cpg: CpgResult,
|
||||
spec: TaintSpec | DominanceSpec | ConfigSpec,
|
||||
*,
|
||||
path: str = "",
|
||||
function: str = "",
|
||||
client: Any | None = None,
|
||||
model: str = "",
|
||||
@@ -90,11 +94,11 @@ def scan_region(
|
||||
parameter_sources: bool = False,
|
||||
) -> list[ModelFinding]:
|
||||
"""Findings for one region and family, ordered strongest first."""
|
||||
answer = _arbitrate(cpg, spec, function=function, parameter_sources=parameter_sources)
|
||||
answer = _arbitrate(cpg, spec, path=path, function=function, parameter_sources=parameter_sources)
|
||||
|
||||
if answer is not None and answer.rung is Rung.ENTAILED:
|
||||
# The graph decided. No model call, and no candidate needs asking about: this region has its finding.
|
||||
return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.ENTAILED, witness=answer.witness)]
|
||||
return [ModelFinding(site=_site(path, function, answer.witness), family=spec.family, rung=Rung.ENTAILED, witness=answer.witness)]
|
||||
|
||||
if answer is not None:
|
||||
# A real flow through a sanitizer. Whether it suffices is the residual, and the model HAS coverage
|
||||
@@ -103,7 +107,9 @@ def scan_region(
|
||||
return []
|
||||
said = _ask(client, model, residual_question(spec, answer.witness, function))
|
||||
if said:
|
||||
return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.CORROBORATED, witness=answer.witness)]
|
||||
return [
|
||||
ModelFinding(site=_site(path, function, answer.witness), family=spec.family, rung=Rung.CORROBORATED, witness=answer.witness)
|
||||
]
|
||||
return []
|
||||
|
||||
# The model resolved nothing. It is SILENT, not negative — so the enumerator's candidates are put to the
|
||||
@@ -194,9 +200,22 @@ def _ask(client: Any, model: str, question: str) -> bool | None:
|
||||
return bool(payload["vulnerable"]) if isinstance(payload, Mapping) and isinstance(payload.get("vulnerable"), bool) else None
|
||||
|
||||
|
||||
def _site(function: str, witness: str) -> str:
|
||||
line = witness.partition("(line ")[2].split(",")[0].strip(") ") if "(line " in witness else "?"
|
||||
return f"{function or '?'}:{line}"
|
||||
_LINE_IN_WITNESS = re.compile(r"\(line (\d+)")
|
||||
|
||||
|
||||
def _site(path: str, function: str, witness: str) -> str:
|
||||
"""``path:line:function`` -- the same shape a candidate id carries, because one consumer parses both.
|
||||
|
||||
An arbitrated finding used to be ``function:line``, which the report layer then read as ``path:line``:
|
||||
every entailed finding arrived with a function name where its file belonged and no line at all. A
|
||||
contributor cannot open that, and nothing downstream can match it against a known location.
|
||||
"""
|
||||
# A regex, because the witness is prose and the delimiter after the number varies: "(line 17,"
|
||||
# in one family and "(line 17):" in another. Splitting on a comma produced sites like
|
||||
# `config.py:17): permissive literal '0.0.0.0':?`.
|
||||
found = _LINE_IN_WITNESS.search(witness)
|
||||
line = found.group(1) if found else "?"
|
||||
return f"{path or '?'}:{line}:{function or '?'}"
|
||||
|
||||
|
||||
__all__ = ["CANDIDATE_QUESTION", "MAX_JUDGED_CANDIDATES", "ModelFinding", "residual_question", "scan_region"]
|
||||
|
||||
@@ -6,10 +6,11 @@ two costs that were free at pair scale dominate here:
|
||||
* **CPG construction.** A build takes 5-30 seconds on a forty-line excerpt. One per region would make a
|
||||
repository scan unusable, so this builds **one** and reuses it across every region.
|
||||
* **Model calls.** The enumerator caps candidates at eight per region, which says nothing useful about a scan
|
||||
with two thousand regions. The budget is a **total for the run**, spent on the highest-ranked regions first,
|
||||
and whatever it does not reach is counted into ``regions_unjudged`` rather than quietly dropped. A tool that
|
||||
silently analyses a tenth of a repository and reports a clean result is worse than one that says what it
|
||||
did not look at.
|
||||
with two thousand regions. The budget is a **total for the run**, spent on the highest-ranked regions first.
|
||||
It bounds model calls and nothing else: when it runs out the JUDGE is withheld and the graph keeps deciding,
|
||||
because arbitration costs nothing. Those regions are counted into ``regions_unasked``, and anything
|
||||
``max_regions`` cut before arbitration into ``regions_unjudged``. A tool that silently analyses a tenth of a
|
||||
repository and reports a clean result is worse than one that says what it did not look at.
|
||||
|
||||
Ordering puts rung before rank: what the model established outranks what the LLM merely proposed, whatever
|
||||
the region's risk score said beforehand.
|
||||
@@ -58,11 +59,24 @@ class ModelScanResult:
|
||||
by_rung: Mapping[str, int] = field(default_factory=dict)
|
||||
regions_scanned: int = 0
|
||||
regions_unjudged: int = 0
|
||||
# Arbitrated by the graph, but with the judge withheld because the budget was spent. Not the same as
|
||||
# unjudged, and not the same as clean.
|
||||
regions_unasked: int = 0
|
||||
model_calls: int = 0
|
||||
cost_usd: float = 0.0
|
||||
seconds: float = 0.0
|
||||
# The split, not just the total. Which stage a scan's wall-clock actually goes to decides real questions
|
||||
# -- whether a warm Joern server would buy anything, whether the budget or the CPG is the ceiling -- and
|
||||
# a single figure cannot answer any of them.
|
||||
build_seconds: float = 0.0
|
||||
query_seconds: float = 0.0
|
||||
degradations: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
@property
|
||||
def arbitrate_seconds(self) -> float:
|
||||
"""What is left once the CPG is built and queried: the judge calls and the arbiters themselves."""
|
||||
return round(max(self.seconds - self.build_seconds - self.query_seconds, 0.0), 2)
|
||||
|
||||
|
||||
def scan_repository(
|
||||
root: Path,
|
||||
@@ -79,12 +93,15 @@ def scan_repository(
|
||||
degradations: list[Mapping[str, object]] = []
|
||||
|
||||
taxonomy = load_families()
|
||||
build_started = time.monotonic()
|
||||
cpg = backend.build(root)
|
||||
build_seconds = round(time.monotonic() - build_started, 2)
|
||||
if cpg is None:
|
||||
return ModelScanResult(
|
||||
by_rung=_empty_tally(),
|
||||
regions_unjudged=len(regions),
|
||||
seconds=round(time.monotonic() - started, 2),
|
||||
build_seconds=build_seconds,
|
||||
degradations=({"stage": "model", "reason": "cpg_build_failed"},),
|
||||
)
|
||||
|
||||
@@ -111,6 +128,7 @@ def scan_repository(
|
||||
# per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while
|
||||
# the queries themselves are milliseconds once the CPG is loaded.
|
||||
rows_by_id: dict[str, list[object]] = {}
|
||||
query_started = time.monotonic()
|
||||
for kind, requests in _grouped(work).items():
|
||||
batch = getattr(cpg, "run_batch", None)
|
||||
if callable(batch):
|
||||
@@ -119,17 +137,23 @@ def scan_repository(
|
||||
for rid, params in requests.items():
|
||||
answered = cpg.run(kind, params)
|
||||
rows_by_id[rid] = list(answered) if isinstance(answered, list) else []
|
||||
query_seconds = round(time.monotonic() - query_started, 2)
|
||||
|
||||
# Phase 3: arbitrate from the rows already in hand. The arbiters are unchanged: each is handed a
|
||||
# CpgResult that simply returns its own prefetched rows.
|
||||
#
|
||||
# `judged` counts regions actually ARBITRATED, not regions collected: a region the budget stopped before
|
||||
# is unjudged, and reporting it as scanned is the silent truncation this exists to prevent.
|
||||
# The budget bounds MODEL CALLS, and nothing else. Arbitration is the CPG alone -- it costs no call, no
|
||||
# token and no money -- so an exhausted budget withholds the JUDGE and lets the graph keep deciding. It
|
||||
# used to break the loop instead, which spent a paid limit to stop unpaid work: on VAmPI that left 16 of
|
||||
# 22 regions unexamined and missed a BOLA the graph entails for free.
|
||||
#
|
||||
# `judged` counts regions actually ARBITRATED and `unasked` those arbitrated with no judge behind them.
|
||||
# Both are reported: a region the graph settled and a region the graph was silent about but nobody could
|
||||
# afford to ask are not the same result, and collapsing them is the silent truncation this prevents.
|
||||
judged: set[tuple[str, str | None]] = set()
|
||||
unasked: set[tuple[str, str | None]] = set()
|
||||
for rid, region, spec in work:
|
||||
if counted is not None and counted.calls >= limits.max_model_calls:
|
||||
degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unjudged": max(len(regions) - len(judged), 0)})
|
||||
break
|
||||
exhausted = counted is not None and counted.calls >= limits.max_model_calls
|
||||
family = getattr(spec, "family", "")
|
||||
prefetched = CpgResult(cpg_path=cpg.cpg_path, run=_prefetched(rows_by_id.get(rid, [])))
|
||||
try:
|
||||
@@ -138,8 +162,9 @@ def scan_repository(
|
||||
found = scan_region(
|
||||
prefetched,
|
||||
spec,
|
||||
path=region.path,
|
||||
function=region.function or "",
|
||||
client=counted,
|
||||
client=None if exhausted else counted,
|
||||
model=model,
|
||||
candidates=candidates,
|
||||
parameter_sources=False, # a repository's parameters are not all attacker input
|
||||
@@ -149,8 +174,13 @@ def scan_repository(
|
||||
degradations.append({"stage": "model", "reason": "region_failed", "path": region.path})
|
||||
continue
|
||||
judged.add((region.path, region.function))
|
||||
if exhausted:
|
||||
unasked.add((region.path, region.function))
|
||||
collected.extend((finding, region.rank) for finding in found)
|
||||
|
||||
if unasked:
|
||||
degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unasked": len(unasked)})
|
||||
|
||||
scanned = len(judged)
|
||||
|
||||
findings = _ordered(collected)
|
||||
@@ -159,9 +189,12 @@ def scan_repository(
|
||||
by_rung=_tally(findings),
|
||||
regions_scanned=scanned,
|
||||
regions_unjudged=max(len(regions) - scanned, 0),
|
||||
regions_unasked=len(unasked),
|
||||
model_calls=counted.calls if counted is not None else 0,
|
||||
cost_usd=round(_cost(counted), 4),
|
||||
seconds=round(time.monotonic() - started, 2),
|
||||
build_seconds=build_seconds,
|
||||
query_seconds=query_seconds,
|
||||
degradations=tuple(degradations),
|
||||
)
|
||||
|
||||
@@ -188,9 +221,9 @@ def _grouped(work: Sequence[tuple[str, ScanRegion, ArbiterSpec]]) -> dict[str, d
|
||||
if isinstance(spec, DominanceSpec):
|
||||
kind, params = "dominance", dominance_params(spec, function=function)
|
||||
elif isinstance(spec, ConfigSpec):
|
||||
kind, params = "config", config_params(spec, function=function)
|
||||
kind, params = "config", config_params(spec, function=function, file=region.path)
|
||||
else:
|
||||
kind, params = "taint", taint_params(spec, function=function, parameter_sources=False)
|
||||
kind, params = "taint", taint_params(spec, function=function, file=region.path, parameter_sources=False)
|
||||
grouped.setdefault(kind, {})[rid] = params
|
||||
return grouped
|
||||
|
||||
|
||||
@@ -19,29 +19,35 @@ from .ladder import Rung, Verdict
|
||||
from .specs import TaintSpec
|
||||
|
||||
|
||||
def request_params(spec: TaintSpec, *, function: str = "", parameter_sources: bool = False) -> dict[str, object]:
|
||||
def request_params(spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False) -> dict[str, object]:
|
||||
"""The query parameters this arbiter sends. Shared with the batcher so there is one definition, not two.
|
||||
|
||||
Two copies of a parameter dict is precisely the duplication that produced five bugs of one shape in the
|
||||
predecessor: a change made in one place and not its sibling.
|
||||
|
||||
``file`` scopes the answer to one source file. A region with no enclosing function sends ``function=""``,
|
||||
which without this matches the whole repository and gets that answer attributed to it -- one permissive
|
||||
literal in `app.py` became eight identical entailed findings in eight files that do not contain it. Empty
|
||||
means unscoped, which is what the single-region pair path still uses.
|
||||
"""
|
||||
return {
|
||||
"sources": spec.sources,
|
||||
"sinks": spec.sinks,
|
||||
"sanitizers": spec.sanitizers,
|
||||
"function": function,
|
||||
"file": file,
|
||||
"parameterSources": "true" if parameter_sources else "false",
|
||||
}
|
||||
|
||||
|
||||
def verdict(cpg: CpgResult, spec: TaintSpec, *, function: str = "", parameter_sources: bool = False) -> Verdict | None:
|
||||
def verdict(cpg: CpgResult, spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False) -> Verdict | None:
|
||||
"""The strongest verdict the taint query supports for ``spec`` in ``function``, or ``None`` to stay at suspicion.
|
||||
|
||||
``parameter_sources`` additionally treats the labeled function's own parameters as untrusted. That is right
|
||||
for a function-level pair, where the function boundary *is* the trust boundary, and wrong for a whole
|
||||
repository, where most parameters carry internal values — so it is off by default and the caller opts in.
|
||||
"""
|
||||
rows = cpg.run("taint", request_params(spec, function=function, parameter_sources=parameter_sources))
|
||||
rows = cpg.run("taint", request_params(spec, function=function, file=file, parameter_sources=parameter_sources))
|
||||
flows = _flows(rows, function=function)
|
||||
# A sink whose *shape* is safe is the fix, not the bug. `execute(sql, params)` binds rather than
|
||||
# interpolates and `printf("literal", x)` has a constant format string, so the flow that reaches them is
|
||||
|
||||
Reference in New Issue
Block a user