diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index eeeba28..6a9f153 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -189,6 +189,18 @@ class JoernBackend: logger.warning("no such cpg query: %s", script) return {} payload = json.dumps({rid: {k: _render(v) for k, v in req.items()} for rid, req in requests.items()}) + # Through a FILE, never the command line. Linux caps a single argument at MAX_ARG_STRLEN (128KB) + # whatever ARG_MAX says, and a repository blows through that: 2500 taint requests over a 637-file + # WordPress plugin is a 1.49MB payload. execve returns E2BIG, `_run` catches the OSError, this returns + # {} -- and {} reads as "no rows found". The scan reported 500 regions examined and nothing found, in + # 0.05 seconds of taint query, which is a quiet failure of exactly the kind the rung ladder cannot + # catch because no verdict was ever produced to label. + requests_file = cpg_path.parent / f"{query}-requests.json" + try: + requests_file.write_text(payload) + except OSError as exc: + logger.warning("could not stage cpg batch %s: %s", query, exc) + return {} command = [ shutil.which("joern") or "joern", self._heap_flag(), @@ -197,7 +209,7 @@ class JoernBackend: "--param", f"cpgFile={cpg_path}", "--param", - f"requests={payload}", + f"requestsFile={requests_file}", ] completed = self._run(command, timeout=self.query_timeout, cwd=cpg_path.parent) if completed is None or completed.returncode != 0: diff --git a/src/openultrasast/cpg/queries/config.sc b/src/openultrasast/cpg/queries/config.sc index e8a8eec..5941afe 100644 --- a/src/openultrasast/cpg/queries/config.sc +++ b/src/openultrasast/cpg/queries/config.sc @@ -14,9 +14,23 @@ // BATCHED, like taint.sc. Single-request form kept for the committed measurements. -@main def exec(cpgFile: String, settings: String = "", function: String = "", file: String = "", requests: String = "") = { +@main def exec( + cpgFile: String, + settings: String = "", + function: String = "", + file: String = "", + requests: String = "", + requestsFile: String = "" +) = { importCpg(cpgFile) + // The batch arrives as a FILE. A single command-line argument is capped at 128KB on Linux and a + // repository's requests are megabytes, so passing them as `--param requests=` failed with E2BIG and the + // driver read the empty result as "no rows". `requests` is kept for the single-request forms and the tests. + val requestsJson = + if (requestsFile.nonEmpty) scala.io.Source.fromFile(requestsFile).mkString + else requests + def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList def rowsFor(settingsS: String, functionS: String, fileS: String): List[ujson.Obj] = { @@ -53,8 +67,8 @@ } println("---OUSAST-CPG-BEGIN---") - if (requests.nonEmpty) { - val parsed = ujson.read(requests).obj + if (requestsJson.nonEmpty) { + val parsed = ujson.read(requestsJson).obj val answers = parsed.map { case (id, req) => def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") id -> ujson.Arr(rowsFor(field("settings"), field("function"), field("file")): _*) diff --git a/src/openultrasast/cpg/queries/dominance.sc b/src/openultrasast/cpg/queries/dominance.sc index 23da0e5..dee665e 100644 --- a/src/openultrasast/cpg/queries/dominance.sc +++ b/src/openultrasast/cpg/queries/dominance.sc @@ -42,10 +42,18 @@ file: String = "", operationRequires: String = "", dischargersByKind: String = "", - requests: String = "" + requests: String = "", + requestsFile: String = "" ) = { importCpg(cpgFile) + // The batch arrives as a FILE. A single command-line argument is capped at 128KB on Linux and a + // repository's requests are megabytes, so passing them as `--param requests=` failed with E2BIG and the + // driver read the empty result as "no rows". `requests` is kept for the single-request forms and the tests. + val requestsJson = + if (requestsFile.nonEmpty) scala.io.Source.fromFile(requestsFile).mkString + else requests + def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList def parseMap(raw: String): Map[String, List[String]] = @@ -162,8 +170,8 @@ } println("---OUSAST-CPG-BEGIN---") - if (requests.nonEmpty) { - val parsed = ujson.read(requests).obj + if (requestsJson.nonEmpty) { + val parsed = ujson.read(requestsJson).obj val answers = parsed.map { case (id, req) => def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") id -> ujson.Arr( diff --git a/src/openultrasast/cpg/queries/taint.sc b/src/openultrasast/cpg/queries/taint.sc index 32e7b3a..3bdb956 100644 --- a/src/openultrasast/cpg/queries/taint.sc +++ b/src/openultrasast/cpg/queries/taint.sc @@ -48,10 +48,18 @@ callDepth: String = "0", boundedSinks: String = "", parameterSources: String = "false", - requests: String = "" + requests: String = "", + requestsFile: String = "" ) = { importCpg(cpgFile) + // The batch arrives as a FILE. A single command-line argument is capped at 128KB on Linux and a + // repository's requests are megabytes, so passing them as `--param requests=` failed with E2BIG and the + // driver read the empty result as "no rows". `requests` is kept for the single-request forms and the tests. + val requestsJson = + if (requestsFile.nonEmpty) scala.io.Source.fromFile(requestsFile).mkString + else requests + def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList def rowsFor( @@ -254,8 +262,8 @@ } println("---OUSAST-CPG-BEGIN---") - if (requests.nonEmpty) { - val parsed = ujson.read(requests).obj + if (requestsJson.nonEmpty) { + val parsed = ujson.read(requestsJson).obj val answers = parsed.map { case (id, req) => def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") val paramSrc = req.obj.get("parameterSources").map(_.str).getOrElse("false") diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index a0f8c61..04ba800 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -303,3 +303,38 @@ def test_an_unknown_language_has_nothing_to_retry_with(tmp_path: Path, monkeypat monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") assert JoernBackend(runner=runner).build(tmp_path, language="cobol") is None assert JoernBackend(runner=runner).build(tmp_path) is None + + +def test_a_batch_goes_through_a_file_not_the_command_line(tmp_path: Path) -> None: + """Linux caps a SINGLE argument at MAX_ARG_STRLEN (128KB) whatever ARG_MAX says. + + 2500 taint requests over a 637-file WordPress plugin is a 1.49MB payload. As `--param requests=` + execve returned E2BIG, the runner caught the OSError, query_batch returned {} -- and {} reads as "no rows + found". The scan reported 500 regions examined, nothing found, and 0.05 seconds of taint query: a quiet + failure the rung ladder cannot catch, because no verdict was ever produced to label. + """ + from openultrasast.cpg.backend import BEGIN, END, JoernBackend + + commands: list[list[str]] = [] + + class _Done: + returncode = 0 + stderr = "" + stdout = f'{BEGIN}\n{{"r0": []}}\n{END}\n' + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + commands.append(list(command)) + return _Done() + + cpg = tmp_path / "cpg.bin" + cpg.write_text("cpg") + big = {f"r{i}": {"sources": ("request.args",) * 40, "sinks": ("execute",) * 40} for i in range(2000)} + + JoernBackend(runner=runner).query_batch(cpg, "taint", big) + + argv = commands[0] + assert not any(len(arg) > 131072 for arg in argv), "no argument may approach MAX_ARG_STRLEN" + staged = [arg for arg in argv if arg.startswith("requestsFile=")] + assert staged, "the batch is staged to a file" + written = Path(staged[0].split("=", 1)[1]) + assert written.is_file() and len(written.read_text()) > 200_000, "and the file holds the whole payload"