mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(pre-push-safety-net): bound backend execution by one deadline
This commit is contained in:
@@ -0,0 +1,63 @@
|
|||||||
|
# Group 3 implementation verification
|
||||||
|
|
||||||
|
Date: 2026-09-13. Scope: tasks 3.1–3.4. The maintainer authorized continuing this
|
||||||
|
group after the group 2 completion and next-step summary. Tasks remain unchecked
|
||||||
|
until independent review and fresh verification establish their completion.
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
This group integrates the existing backend and ranker with generic execution,
|
||||||
|
change-context and scope contracts. It does not implement new detector semantics,
|
||||||
|
alert admission, persistent caching, or hook installation. Equivalent normalized
|
||||||
|
PHP and JavaScript evidence must retain equivalent core decisions. Vendor sources
|
||||||
|
must remain outside both graph inputs and targets; missing external semantics are
|
||||||
|
coverage limits.
|
||||||
|
|
||||||
|
## Verification state
|
||||||
|
|
||||||
|
Implementation is in progress. No group-level completion claim is made yet.
|
||||||
|
|
||||||
|
## Baseline
|
||||||
|
|
||||||
|
Groups 1 and 2 completed at commit `948ee02`; the working tree was clean before
|
||||||
|
this group began. The prior final suite recorded 1,069 passing and nine skipped
|
||||||
|
tests. Three pre-existing mypy diagnostics concerned reuse of `candidates` in
|
||||||
|
`model/scan.py`; fresh checks will report whether they remain.
|
||||||
|
|
||||||
|
Historical Joern query failures cannot be declared resolved merely because a
|
||||||
|
bounded failure returns promptly. Runtime smoke establishes readable source,
|
||||||
|
nonempty graph census and query payloads for its declared small fixtures. It does
|
||||||
|
not establish full-workload detection, actionable precision or hook latency.
|
||||||
|
|
||||||
|
## Task 3.1 evidence
|
||||||
|
|
||||||
|
The backend accepts an optional generic `ExecutionBudget`. Each build binds an
|
||||||
|
independent session to its graph callbacks; probes, builds, overlays, census,
|
||||||
|
queries and retries consume the same remaining time. Process startup also counts.
|
||||||
|
A timed-out stage prevents further retries. The budgeted process path kills the
|
||||||
|
original process group even after its launcher exits, bounds reaping and owned
|
||||||
|
scratch cleanup, and exposes incomplete cleanup explicitly. The driver releases
|
||||||
|
graph ownership on exceptions as well as normal return. Legacy backends cannot
|
||||||
|
silently discard a supplied deadline; optional unbounded judge calls are withheld
|
||||||
|
with an explicit reason. Ordinary scans retain their existing budget behavior.
|
||||||
|
|
||||||
|
`benchmarks/measurements/2026-09-13-engine-budget-smoke.json` records the locally
|
||||||
|
rebuilt non-root image and exact packaged backend/driver hashes. Under one
|
||||||
|
900-second lab budget, native PHP read 45 source bytes and JavaScript read 48;
|
||||||
|
both produced populated graphs, dataflow overlays and exact source-node witnesses.
|
||||||
|
Unreadable PHP source was explicitly refused. PHP took 73.620 seconds and JavaScript
|
||||||
|
34.756 seconds in this small smoke; these are runtime checks, not hook latency.
|
||||||
|
|
||||||
|
An independent controlled process probe read its 241-byte input script, launched
|
||||||
|
a parent and child ignoring SIGTERM, and observed termination in 0.804 seconds
|
||||||
|
under a 0.8-second deadline plus 0.8-second cancellation allowance. Signal delivery
|
||||||
|
may briefly leave a child visible immediately after return; the probe polls only
|
||||||
|
within the original allowance. Zombies count as terminated, not running descendants.
|
||||||
|
Concrete reusable graph leases and persisted completion records remain later tasks.
|
||||||
|
|
||||||
|
The three former mypy errors were resolved by renaming the evidence-region list
|
||||||
|
inside the driver; this does not change the ranker. Global mypy now checks all
|
||||||
|
108 source files successfully. Independent review APPROVED task 3.1. Its fresh final suite passed: 1,089 tests,
|
||||||
|
nine skipped, 96.85 seconds; global Ruff, formatting (219 files), mypy and diff
|
||||||
|
checks passed. Verification status: VERIFIED, claim type TASK. This completes
|
||||||
|
8/27 tasks; tasks 3.2–3.4 remain in progress for this group.
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"feature_name": "pre-push-safety-net",
|
"feature_name": "pre-push-safety-net",
|
||||||
"created_at": "2026-09-12T15:35:00Z",
|
"created_at": "2026-09-12T15:35:00Z",
|
||||||
"updated_at": "2026-09-12T19:09:25.547910Z",
|
"updated_at": "2026-09-13T08:02:59.282414+00:00",
|
||||||
"language": "en",
|
"language": "en",
|
||||||
"phase": "implementation",
|
"phase": "implementation",
|
||||||
"approvals": {
|
"approvals": {
|
||||||
@@ -19,7 +19,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"ready_for_implementation": true,
|
"ready_for_implementation": true,
|
||||||
"implementation_approval_note": "Groups 1 and 2 explicitly authorized by the maintainer and verified complete after independent review. Remaining task plan retains approval, with execution sequenced by the spec. Implementation of the complete hook is still pending.",
|
"implementation_approval_note": "Groups 1 and 2 verified complete. Maintainer authorized continuing group 3; remaining approved tasks execute in dependency order with independent review and fresh verification.",
|
||||||
"depends_on": [
|
"depends_on": [
|
||||||
"contributor-scan"
|
"contributor-scan"
|
||||||
],
|
],
|
||||||
@@ -45,6 +45,11 @@
|
|||||||
"at": "2026-09-12T18:32:19.033933Z",
|
"at": "2026-09-12T18:32:19.033933Z",
|
||||||
"source": "ok, next kiro phase, approved; correct. we can continue with the spec",
|
"source": "ok, next kiro phase, approved; correct. we can continue with the spec",
|
||||||
"action": "Authorized group 2 (2.1\u20132.3), resumed after local Claude Mem repair. Independent review and fresh verification remain required."
|
"action": "Authorized group 2 (2.1\u20132.3), resumed after local Claude Mem repair. Independent review and fresh verification remain required."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-09-13T07:48:02.155687Z",
|
||||||
|
"source": "continue with the task",
|
||||||
|
"action": "Authorized continuing group 3 (3.1\u20133.4) after the group 3 next-step summary. Independent task review and fresh verification remain required."
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"task_review": {
|
"task_review": {
|
||||||
@@ -64,13 +69,15 @@
|
|||||||
"1.4",
|
"1.4",
|
||||||
"2.1",
|
"2.1",
|
||||||
"2.2",
|
"2.2",
|
||||||
"2.3"
|
"2.3",
|
||||||
|
"3.1"
|
||||||
],
|
],
|
||||||
"remaining_tasks": 20,
|
"remaining_tasks": 19,
|
||||||
"verification_record": "implementation-group-2.md",
|
"verification_record": "implementation-group-3.md",
|
||||||
"verification_records": [
|
"verification_records": [
|
||||||
"implementation-group-1.md",
|
"implementation-group-1.md",
|
||||||
"implementation-group-2.md"
|
"implementation-group-2.md",
|
||||||
|
"implementation-group-3.md"
|
||||||
],
|
],
|
||||||
"next_group": 3
|
"next_group": 3
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Implementation Plan: pre-push-safety-net
|
# Implementation Plan: pre-push-safety-net
|
||||||
|
|
||||||
Status: task plan approved by the maintainer's "approve." on 2026-09-12. Groups 1 and 2 (tasks 1.1–2.3)
|
Status: task plan approved by the maintainer's "approve." on 2026-09-12. Groups 1 and 2 (tasks 1.1–2.3)
|
||||||
are complete after independent review and fresh verification. Groups 3–8 remain pending;
|
are complete after independent review and fresh verification. Group 3 is in progress (3.1 verified); groups 4–8 remain pending;
|
||||||
the hook and capability admission are not yet implemented. See `implementation-group-1.md` and `implementation-group-2.md`.
|
the hook and capability admission are not yet implemented. See `implementation-group-1.md` and `implementation-group-2.md`.
|
||||||
|
|
||||||
## Execution contract
|
## Execution contract
|
||||||
@@ -79,13 +79,14 @@ the hook and capability admission are not yet implemented. See `implementation-g
|
|||||||
- _Verified 2026-09-12:_ independent review APPROVED; fresh final suite 1,069 passed, 9 skipped; focused 148 passed. Global Ruff/format, touched mypy and regression gates passed. PHP/JavaScript fixtures retain removed spans and unchanged-operation anchors across declaration renames; uncertain moves, repeated lines, binary and unavailable inputs remain unresolved. Forty independent real-Git comparisons checked exact spans/anchors. Network-disabled image smoke preserved committed PHP/JavaScript bytes and dirty non-HEAD state, with cleanup verified; see `implementation-group-2.md`. Lexical anchors do not establish semantic impact or novelty.
|
- _Verified 2026-09-12:_ independent review APPROVED; fresh final suite 1,069 passed, 9 skipped; focused 148 passed. Global Ruff/format, touched mypy and regression gates passed. PHP/JavaScript fixtures retain removed spans and unchanged-operation anchors across declaration renames; uncertain moves, repeated lines, binary and unavailable inputs remain unresolved. Forty independent real-Git comparisons checked exact spans/anchors. Network-disabled image smoke preserved committed PHP/JavaScript bytes and dirty non-HEAD state, with cleanup verified; see `implementation-group-2.md`. Lexical anchors do not establish semantic impact or novelty.
|
||||||
|
|
||||||
- [ ] 3. Integrate bounded execution and ranker scope with the existing engine
|
- [ ] 3. Integrate bounded execution and ranker scope with the existing engine
|
||||||
- [ ] 3.1 (P) Enforce one remaining-time budget in the backend
|
- [x] 3.1 (P) Enforce one remaining-time budget in the backend
|
||||||
- Thread the absolute deadline through readability probes, frontend builds, overlays, census, query batches, fallbacks and retries; no phase gets a fresh full budget.
|
- Thread the absolute deadline through readability probes, frontend builds, overlays, census, query batches, fallbacks and retries; no phase gets a fresh full budget.
|
||||||
- Bound process-group termination and lease/scratch cleanup by the cancellation allowance, replacing any longer fixed cleanup waits on the push path.
|
- Bound process-group termination and lease/scratch cleanup by the cancellation allowance, replacing any longer fixed cleanup waits on the push path.
|
||||||
- Done when controlled hanging subprocesses at build/overlay/query stages terminate without orphan children inside the declared allowance, while ordinary scans preserve their prior behavior.
|
- Done when controlled hanging subprocesses at build/overlay/query stages terminate without orphan children inside the declared allowance, while ordinary scans preserve their prior behavior.
|
||||||
- _Boundary: Cache and Engine Boundary, generic driver integration_
|
- _Boundary: Cache and Engine Boundary, generic driver integration_
|
||||||
- _Depends: 1.4_
|
- _Depends: 1.4_
|
||||||
- _Requirements: 4.1, 4.2, 7.3_
|
- _Requirements: 4.1, 4.2, 7.3_
|
||||||
|
- _Verified 2026-09-13:_ independent review APPROVED; fresh suite 1,089 passed, nine skipped; global Ruff/format and mypy (108 files) passed. Real hanging probe/frontend/overlay/census/query processes obey the shared deadline and cancellation allowance; retries do not reset it. Non-root packaged PHP/JavaScript graph/query smoke passed with matching source hashes under one 900-second lab budget. See `implementation-group-3.md` and `benchmarks/measurements/2026-09-13-engine-budget-smoke.json`. Runtime readiness only; no hook latency claim. Former three mypy errors resolved by a behavior-preserving variable rename.
|
||||||
- [ ] 3.2 Return the ranker's exact selected and deferred work
|
- [ ] 3.2 Return the ranker's exact selected and deferred work
|
||||||
- Integrate the existing ranker through explicit mode selection and shared question identity; record selected/deferred IDs, evidence and reasons directly from the final order.
|
- Integrate the existing ranker through explicit mode selection and shared question identity; record selected/deferred IDs, evidence and reasons directly from the final order.
|
||||||
- Preserve unanswered evidence as unknown rather than tier zero; report per-family coverage so taint ranking cannot masquerade as authorization/configuration coverage.
|
- Preserve unanswered evidence as unknown rather than tier zero; report per-family coverage so taint ranking cannot masquerade as authorization/configuration coverage.
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
{
|
||||||
|
"recorded_at": "2026-09-13T08:00:44.226437+00:00",
|
||||||
|
"task": "3.1",
|
||||||
|
"image_id": "sha256:f422c603db9f3a1f5e33f255b36157ac0d660b05a3845eb8b7bb87f1fb265ce7",
|
||||||
|
"build_exit_code": 0,
|
||||||
|
"smoke_exit_code": 0,
|
||||||
|
"packaged_source_hashes": {
|
||||||
|
"openultrasast.cpg.backend": "2e327d26958efd592932bbe5a84aa122d969c0e49a1c13469a4d956b621a9a34",
|
||||||
|
"openultrasast.model.scan": "3ceed02c315893c5e54b1e175faaf261eee3883fd20ca85a013ffc9d9a6650f2"
|
||||||
|
},
|
||||||
|
"packaged_sources_match_workspace": true,
|
||||||
|
"smoke_script_sha256": "710a54e2a208bb71263771192d67a82e934526632e88f6aa626de9950fa80128",
|
||||||
|
"runtime": {
|
||||||
|
"joern_version": "4.0.625",
|
||||||
|
"php_version": "PHP 8.3.6 (cli) (built: Sep 2 2026 12:56:02) (NTS)",
|
||||||
|
"php_parser_bytes": 43,
|
||||||
|
"purpose": "runtime and query smoke; does not qualify vulnerability detection or hook latency",
|
||||||
|
"shared_budget_seconds": 900.0,
|
||||||
|
"languages": {
|
||||||
|
"php": {
|
||||||
|
"frontend": "php2cpg",
|
||||||
|
"source_bytes": 45,
|
||||||
|
"source_sha256": "f744d636071c3e5a8e72ef33fa746db22e705dfdc66dba98b5238875054d801a",
|
||||||
|
"graph_bytes": 49509,
|
||||||
|
"census": {
|
||||||
|
"files": "2",
|
||||||
|
"methods": "3",
|
||||||
|
"overlays": "base,controlflow,typerel,callgraph,dataflowOss",
|
||||||
|
"maxHeapMB": "1024"
|
||||||
|
},
|
||||||
|
"source_witness": {
|
||||||
|
"files": [
|
||||||
|
"probe.php",
|
||||||
|
"<unknown>"
|
||||||
|
],
|
||||||
|
"handlers": 1,
|
||||||
|
"evalCalls": 1
|
||||||
|
},
|
||||||
|
"elapsed_seconds": 73.62
|
||||||
|
},
|
||||||
|
"javascript": {
|
||||||
|
"frontend": "jssrc2cpg",
|
||||||
|
"source_bytes": 48,
|
||||||
|
"source_sha256": "10c0c21a6e2f988119f305809af1f98e1ff39fecb9f40d6cd3dfa05301e0015b",
|
||||||
|
"graph_bytes": 71165,
|
||||||
|
"census": {
|
||||||
|
"files": "1",
|
||||||
|
"methods": "5",
|
||||||
|
"overlays": "base,controlflow,typerel,callgraph,dataflowOss",
|
||||||
|
"maxHeapMB": "1024"
|
||||||
|
},
|
||||||
|
"source_witness": {
|
||||||
|
"files": [
|
||||||
|
"probe.js"
|
||||||
|
],
|
||||||
|
"handlers": 1,
|
||||||
|
"evalCalls": 1
|
||||||
|
},
|
||||||
|
"elapsed_seconds": 34.756
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"unreadable_source": {
|
||||||
|
"status": "explicit_failure",
|
||||||
|
"reason": "the php interpreter cannot read /tmp/ousast-engine-smoke-npgsdp9_/php/probe.php, so the frontend would parse nothing and write an empty graph. Check that php is a real interpreter with access to this tree -- a container or wrapper without the right mount fails exactly this way."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"controlled_process_probe": {
|
||||||
|
"source_bytes": 241,
|
||||||
|
"source_sha256": "a7c6471185a8a05a8b812fa8dfb878dde321aefca03ad046d3bc5465d2cf4c14",
|
||||||
|
"deadline_seconds": 0.8,
|
||||||
|
"cancellation_allowance_seconds": 0.8,
|
||||||
|
"elapsed_seconds": 0.8036922400351614,
|
||||||
|
"process_states": [
|
||||||
|
"absent",
|
||||||
|
"Z"
|
||||||
|
],
|
||||||
|
"result": "explicit_failure",
|
||||||
|
"purpose": "controlled subprocess termination; not Joern detection or hook latency"
|
||||||
|
},
|
||||||
|
"limits": [
|
||||||
|
"Small runtime fixtures, not full-workload detection or hook latency.",
|
||||||
|
"Terminated zombie processes are not running descendants; no claim of reaping grandchildren.",
|
||||||
|
"900-second lab budget, not the 30-second hook acceptance target."
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -85,6 +85,11 @@ PHP file. It prints JSON only after all checks pass; a failed build/query raises
|
|||||||
exit. Versions, source hashes/byte counts and timings describe this small runtime smoke, not detection
|
exit. Versions, source hashes/byte counts and timings describe this small runtime smoke, not detection
|
||||||
quality or a pre-push latency guarantee. Keep the image's default non-root user for the permission test.
|
quality or a pre-push latency guarantee. Keep the image's default non-root user for the permission test.
|
||||||
|
|
||||||
|
To exercise the same smoke through one shared backend deadline, add
|
||||||
|
`-e OUSAST_ENGINE_SMOKE_BUDGET_SECONDS=900` to the Docker command. Both language builds and
|
||||||
|
all census/witness queries consume that same budget. The generous smoke allowance verifies
|
||||||
|
runtime integration; it does not represent the hook's target latency.
|
||||||
|
|
||||||
Why this exists: Joern was first installed here by hand with `curl` and `unzip`, which is fine once and
|
Why this exists: Joern was first installed here by hand with `curl` and `unzip`, which is fine once and
|
||||||
unreproducible thereafter. The engine decides whether a finding is a `suspicion` or a `model_entailed`, and
|
unreproducible thereafter. The engine decides whether a finding is a `suspicion` or a `model_entailed`, and
|
||||||
every ceiling number in `benchmarks/measurements` is only comparable against a fixed version — so the version
|
every ceiling number in `benchmarks/measurements` is only comparable against a fixed version — so the version
|
||||||
|
|||||||
+8
-2
@@ -13,6 +13,7 @@ import time
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from openultrasast.cpg.backend import DATAFLOW_OVERLAY, JoernBackend, joern_version
|
from openultrasast.cpg.backend import DATAFLOW_OVERLAY, JoernBackend, joern_version
|
||||||
|
from openultrasast.model.contracts import ExecutionBudget
|
||||||
|
|
||||||
|
|
||||||
def require(condition: object, message: str) -> None:
|
def require(condition: object, message: str) -> None:
|
||||||
@@ -22,6 +23,8 @@ def require(condition: object, message: str) -> None:
|
|||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
require(os.getuid() != 0, "run as the image's non-root user so unreadable-file checks are meaningful")
|
require(os.getuid() != 0, "run as the image's non-root user so unreadable-file checks are meaningful")
|
||||||
|
budget_seconds = float(os.environ.get("OUSAST_ENGINE_SMOKE_BUDGET_SECONDS", "0"))
|
||||||
|
execution_budget = ExecutionBudget(time.monotonic() + budget_seconds, 2.0) if budget_seconds > 0 else None
|
||||||
version = joern_version()
|
version = joern_version()
|
||||||
require(version, "installed Joern version is unknown")
|
require(version, "installed Joern version is unknown")
|
||||||
php = Path(shutil.which("php") or "/missing-php").resolve()
|
php = Path(shutil.which("php") or "/missing-php").resolve()
|
||||||
@@ -39,6 +42,7 @@ def main() -> None:
|
|||||||
"php_version": subprocess.run([str(php), "-v"], check=True, capture_output=True, text=True, timeout=15).stdout.splitlines()[0],
|
"php_version": subprocess.run([str(php), "-v"], check=True, capture_output=True, text=True, timeout=15).stdout.splitlines()[0],
|
||||||
"php_parser_bytes": php_bytes(parser),
|
"php_parser_bytes": php_bytes(parser),
|
||||||
"purpose": "runtime and query smoke; does not qualify vulnerability detection or hook latency",
|
"purpose": "runtime and query smoke; does not qualify vulnerability detection or hook latency",
|
||||||
|
"shared_budget_seconds": budget_seconds or None,
|
||||||
"languages": {},
|
"languages": {},
|
||||||
}
|
}
|
||||||
languages = report["languages"]
|
languages = report["languages"]
|
||||||
@@ -72,7 +76,7 @@ def main() -> None:
|
|||||||
if language == "php":
|
if language == "php":
|
||||||
require(php_bytes(source) == len(data), "PHP and host source byte counts differ")
|
require(php_bytes(source) == len(data), "PHP and host source byte counts differ")
|
||||||
print(f"{language}: opened {filename}, {len(data)} bytes; building graph", file=sys.stderr, flush=True)
|
print(f"{language}: opened {filename}, {len(data)} bytes; building graph", file=sys.stderr, flush=True)
|
||||||
backend = JoernBackend(build_timeout=180, query_timeout=180, heap_mb=1024)
|
backend = JoernBackend(build_timeout=180, query_timeout=180, heap_mb=1024, execution_budget=execution_budget)
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
graph = backend.build(source_root, language=language)
|
graph = backend.build(source_root, language=language)
|
||||||
require(graph, f"{language} graph build failed: {backend.last_failure}")
|
require(graph, f"{language} graph build failed: {backend.last_failure}")
|
||||||
@@ -84,7 +88,9 @@ def main() -> None:
|
|||||||
assert isinstance(census, dict)
|
assert isinstance(census, dict)
|
||||||
require(int(census["files"]) > 0 and int(census["methods"]) > 0, f"empty {language} census")
|
require(int(census["files"]) > 0 and int(census["methods"]) > 0, f"empty {language} census")
|
||||||
require(DATAFLOW_OVERLAY in str(census["overlays"]).split(","), f"{language} missing dataflow overlay: {census}")
|
require(DATAFLOW_OVERLAY in str(census["overlays"]).split(","), f"{language} missing dataflow overlay: {census}")
|
||||||
witness = JoernBackend(queries_dir=queries, query_timeout=180, heap_mb=1024).query(graph.cpg_path, "source_witness", {})
|
witness = JoernBackend(queries_dir=queries, query_timeout=180, heap_mb=1024, execution_budget=execution_budget).query(
|
||||||
|
graph.cpg_path, "source_witness", {}
|
||||||
|
)
|
||||||
require(isinstance(witness, dict), f"{language} source witness unanswered")
|
require(isinstance(witness, dict), f"{language} source witness unanswered")
|
||||||
assert isinstance(witness, dict)
|
assert isinstance(witness, dict)
|
||||||
require(any(str(path).endswith(filename) for path in witness["files"]), f"{language} source absent from graph")
|
require(any(str(path).endswith(filename) for path in witness["files"]), f"{language} source absent from graph")
|
||||||
|
|||||||
@@ -25,14 +25,17 @@ import os
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import signal
|
import signal
|
||||||
|
import stat
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from collections.abc import Callable, Mapping, Sequence
|
from collections.abc import Callable, Mapping, Sequence
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Protocol
|
from typing import Protocol
|
||||||
|
|
||||||
|
from ..model.contracts import ExecutionBudget
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
QUERIES_DIR = Path(__file__).resolve().parent / "queries"
|
QUERIES_DIR = Path(__file__).resolve().parent / "queries"
|
||||||
@@ -239,6 +242,7 @@ class CpgResult:
|
|||||||
# request files and joern's own workspace copy; nothing reclaimed it, and a day of scanning left 1,055
|
# request files and joern's own workspace copy; nothing reclaimed it, and a day of scanning left 1,055
|
||||||
# directories and 355MB behind on a disk that was already at 95%. The driver disposes in a `finally`.
|
# directories and 355MB behind on a disk that was already at 95%. The driver disposes in a `finally`.
|
||||||
cleanup: Callable[[], None] | None = None
|
cleanup: Callable[[], None] | None = None
|
||||||
|
execution_diagnostics: Callable[[], tuple[str, ...]] | None = None
|
||||||
|
|
||||||
|
|
||||||
class CpgBackend(Protocol):
|
class CpgBackend(Protocol):
|
||||||
@@ -272,13 +276,41 @@ class JoernBackend:
|
|||||||
queries_dir: Path = field(default_factory=lambda: QUERIES_DIR)
|
queries_dir: Path = field(default_factory=lambda: QUERIES_DIR)
|
||||||
# Why the last build refused, so the driver can name it instead of reporting a bare `cpg_build_failed`.
|
# Why the last build refused, so the driver can name it instead of reporting a bare `cpg_build_failed`.
|
||||||
last_failure: str = ""
|
last_failure: str = ""
|
||||||
|
execution_budget: ExecutionBudget | None = None
|
||||||
|
_diagnostics: list[str] = field(default_factory=list, init=False, repr=False)
|
||||||
|
_scratch: list[Path] = field(default_factory=list, init=False, repr=False)
|
||||||
|
_cancel_deadline: float | None = field(default=None, init=False, repr=False)
|
||||||
|
|
||||||
def available(self) -> bool:
|
def available(self) -> bool:
|
||||||
from .capability import has_cpg
|
from .capability import has_cpg
|
||||||
|
|
||||||
return has_cpg()
|
return has_cpg()
|
||||||
|
|
||||||
def build(self, root: Path, *, language: str = "", exclude: Sequence[str] = ()) -> CpgResult | None:
|
def build(
|
||||||
|
self, root: Path, *, language: str = "", exclude: Sequence[str] = (), execution_budget: ExecutionBudget | None = None
|
||||||
|
) -> CpgResult | None:
|
||||||
|
"""Bind an independent graph session to the caller's absolute transaction deadline."""
|
||||||
|
budget = execution_budget if execution_budget is not None else self.execution_budget
|
||||||
|
if budget is None:
|
||||||
|
return self._build_impl(root, language=language, exclude=exclude)
|
||||||
|
session = replace(self, execution_budget=budget, last_failure="")
|
||||||
|
result = None
|
||||||
|
try:
|
||||||
|
session._checkpoint()
|
||||||
|
result = session._build_impl(root, language=language, exclude=exclude)
|
||||||
|
session._checkpoint()
|
||||||
|
return result
|
||||||
|
except _DeadlineExpired:
|
||||||
|
result = None
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
if result is None:
|
||||||
|
for scratch in session._scratch:
|
||||||
|
session._cleanup(scratch, final=True)
|
||||||
|
self.last_failure = session.last_failure
|
||||||
|
self._diagnostics = list(session._diagnostics)
|
||||||
|
|
||||||
|
def _build_impl(self, root: Path, *, language: str = "", exclude: Sequence[str] = ()) -> CpgResult | None:
|
||||||
"""Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses.
|
"""Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses.
|
||||||
|
|
||||||
``language`` lets a failed ``joern-parse`` retry through the frontend directly, which is what Joern
|
``language`` lets a failed ``joern-parse`` retry through the frontend directly, which is what Joern
|
||||||
@@ -287,10 +319,13 @@ class JoernBackend:
|
|||||||
parse = shutil.which("joern-parse")
|
parse = shutil.which("joern-parse")
|
||||||
if parse is None and os.environ.get("OPENULTRASAST_JOERN_PROBE", "").strip().lower() not in {"1", "on", "true", "yes"}:
|
if parse is None and os.environ.get("OPENULTRASAST_JOERN_PROBE", "").strip().lower() not in {"1", "on", "true", "yes"}:
|
||||||
return None
|
return None
|
||||||
|
if self.execution_budget is None:
|
||||||
_sweep_stale_scratch()
|
_sweep_stale_scratch()
|
||||||
scratch = Path(tempfile.mkdtemp(prefix=SCRATCH_PREFIX))
|
scratch = Path(tempfile.mkdtemp(prefix=SCRATCH_PREFIX))
|
||||||
|
if self.execution_budget is not None:
|
||||||
|
self._scratch.append(scratch)
|
||||||
cpg_path = scratch / "cpg.bin"
|
cpg_path = scratch / "cpg.bin"
|
||||||
dispose = lambda: shutil.rmtree(scratch, ignore_errors=True) # noqa: E731 -- one expression, named
|
dispose = lambda: self._cleanup(scratch, final=True) # noqa: E731 -- one expression, named
|
||||||
|
|
||||||
# THE PRECONDITION. Before anything is built, make the toolchain open one file from the tree and say
|
# THE PRECONDITION. Before anything is built, make the toolchain open one file from the tree and say
|
||||||
# how big it is. Every instrument failure this project has had takes the same shape -- the input was
|
# how big it is. Every instrument failure this project has had takes the same shape -- the input was
|
||||||
@@ -307,6 +342,7 @@ class JoernBackend:
|
|||||||
if unreadable:
|
if unreadable:
|
||||||
logger.error("cpg build refused for %s: %s", root, unreadable)
|
logger.error("cpg build refused for %s: %s", root, unreadable)
|
||||||
self.last_failure = unreadable
|
self.last_failure = unreadable
|
||||||
|
dispose()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# A frontend whose failures are visible and retryable is worth more than one whose are not.
|
# A frontend whose failures are visible and retryable is worth more than one whose are not.
|
||||||
@@ -329,6 +365,7 @@ class JoernBackend:
|
|||||||
run_batch=lambda query, requests: self.query_batch_across(shards, query, requests),
|
run_batch=lambda query, requests: self.query_batch_across(shards, query, requests),
|
||||||
unparsed=unparsed,
|
unparsed=unparsed,
|
||||||
cleanup=dispose,
|
cleanup=dispose,
|
||||||
|
execution_diagnostics=lambda: tuple(self._diagnostics),
|
||||||
)
|
)
|
||||||
command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)]
|
command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)]
|
||||||
completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
|
completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
|
||||||
@@ -350,6 +387,7 @@ class JoernBackend:
|
|||||||
run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests),
|
run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests),
|
||||||
unparsed=unparsed,
|
unparsed=unparsed,
|
||||||
cleanup=dispose,
|
cleanup=dispose,
|
||||||
|
execution_diagnostics=lambda: tuple(self._diagnostics),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _files_too_large_to_flow(self, root: Path, language: str) -> tuple[str, ...]:
|
def _files_too_large_to_flow(self, root: Path, language: str) -> tuple[str, ...]:
|
||||||
@@ -358,7 +396,7 @@ class JoernBackend:
|
|||||||
if not extensions:
|
if not extensions:
|
||||||
return ()
|
return ()
|
||||||
found: list[str] = []
|
found: list[str] = []
|
||||||
for path in sorted(root.rglob("*")):
|
for path in self._paths(root, "*"):
|
||||||
if path.suffix.lower() not in extensions or not path.is_file():
|
if path.suffix.lower() not in extensions or not path.is_file():
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
@@ -381,7 +419,7 @@ class JoernBackend:
|
|||||||
if version is not None and version >= CLOSURE_DEFECT_FIXED_IN:
|
if version is not None and version >= CLOSURE_DEFECT_FIXED_IN:
|
||||||
return () # fixed upstream; nothing to exclude and every file to keep
|
return () # fixed upstream; nothing to exclude and every file to keep
|
||||||
found: list[str] = []
|
found: list[str] = []
|
||||||
for path in sorted(root.rglob("*.php")):
|
for path in self._paths(root, "*.php"):
|
||||||
if not path.is_file():
|
if not path.is_file():
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
@@ -394,6 +432,8 @@ class JoernBackend:
|
|||||||
continue
|
continue
|
||||||
depth, index = 0, opening
|
depth, index = 0, opening
|
||||||
while index < len(text):
|
while index < len(text):
|
||||||
|
if index % 4096 == 0:
|
||||||
|
self._checkpoint()
|
||||||
if text[index] == "{":
|
if text[index] == "{":
|
||||||
depth += 1
|
depth += 1
|
||||||
elif text[index] == "}":
|
elif text[index] == "}":
|
||||||
@@ -421,7 +461,7 @@ class JoernBackend:
|
|||||||
binary = shutil.which(interpreter)
|
binary = shutil.which(interpreter)
|
||||||
if binary is None:
|
if binary is None:
|
||||||
return "" # absent is a different failure, and the frontend reports it plainly
|
return "" # absent is a different failure, and the frontend reports it plainly
|
||||||
sample = next((path for path in sorted(root.rglob("*")) if path.suffix.lower() in extensions and path.is_file()), None)
|
sample = next((path for path in self._paths(root, "*") if path.suffix.lower() in extensions and path.is_file()), None)
|
||||||
if sample is None:
|
if sample is None:
|
||||||
return "" # nothing of this language to read; not the interpreter's fault
|
return "" # nothing of this language to read; not the interpreter's fault
|
||||||
probe = "$f = $argv[1]; if (!is_readable($f) || filesize($f) < 1) { exit(3); } exit(0);"
|
probe = "$f = $argv[1]; if (!is_readable($f) || filesize($f) < 1) { exit(3); } exit(0);"
|
||||||
@@ -545,7 +585,7 @@ class JoernBackend:
|
|||||||
extensions = _LANGUAGE_EXTENSIONS.get(language.lower())
|
extensions = _LANGUAGE_EXTENSIONS.get(language.lower())
|
||||||
if not extensions:
|
if not extensions:
|
||||||
return None
|
return None
|
||||||
expected = sum(1 for path in root.rglob("*") if path.suffix.lower() in extensions and path.is_file())
|
expected = sum(1 for path in self._paths(root, "*") if path.suffix.lower() in extensions and path.is_file())
|
||||||
if not expected:
|
if not expected:
|
||||||
return (0, 0) # nothing to count is not a failed count: an empty tree's graph is complete
|
return (0, 0) # nothing to count is not a failed count: an empty tree's graph is complete
|
||||||
payload = self.query(cpg_path, "census", {})
|
payload = self.query(cpg_path, "census", {})
|
||||||
@@ -598,6 +638,7 @@ class JoernBackend:
|
|||||||
island = scratch / "excluded-root"
|
island = scratch / "excluded-root"
|
||||||
made = 0
|
made = 0
|
||||||
for name in files:
|
for name in files:
|
||||||
|
self._checkpoint()
|
||||||
source = Path(name)
|
source = Path(name)
|
||||||
if not source.is_absolute():
|
if not source.is_absolute():
|
||||||
source = root / name
|
source = root / name
|
||||||
@@ -624,6 +665,9 @@ class JoernBackend:
|
|||||||
for shard in shards:
|
for shard in shards:
|
||||||
rows = self.query(shard, query, params)
|
rows = self.query(shard, query, params)
|
||||||
if rows is None:
|
if rows is None:
|
||||||
|
if self.execution_budget is not None:
|
||||||
|
self._note("shard_query_incomplete")
|
||||||
|
return None
|
||||||
continue
|
continue
|
||||||
answered += 1
|
answered += 1
|
||||||
if isinstance(rows, list):
|
if isinstance(rows, list):
|
||||||
@@ -645,6 +689,9 @@ class JoernBackend:
|
|||||||
for shard in shards:
|
for shard in shards:
|
||||||
one = self.query_batch(shard, query, requests)
|
one = self.query_batch(shard, query, requests)
|
||||||
if one is None:
|
if one is None:
|
||||||
|
if self.execution_budget is not None:
|
||||||
|
self._note("shard_query_incomplete")
|
||||||
|
return None
|
||||||
continue
|
continue
|
||||||
answered += 1
|
answered += 1
|
||||||
census = one.pop("__census__", None)
|
census = one.pop("__census__", None)
|
||||||
@@ -692,7 +739,7 @@ class JoernBackend:
|
|||||||
saved.replace(cpg_path)
|
saved.replace(cpg_path)
|
||||||
return True
|
return True
|
||||||
finally:
|
finally:
|
||||||
shutil.rmtree(workspace, ignore_errors=True)
|
self._cleanup(workspace)
|
||||||
|
|
||||||
def _build_with_retries(
|
def _build_with_retries(
|
||||||
self, root: Path, cpg_path: Path, scratch: Path, language: str, exclude: Sequence[str] = ()
|
self, root: Path, cpg_path: Path, scratch: Path, language: str, exclude: Sequence[str] = ()
|
||||||
@@ -716,6 +763,7 @@ class JoernBackend:
|
|||||||
"""
|
"""
|
||||||
best: tuple[str, ...] | None = None
|
best: tuple[str, ...] | None = None
|
||||||
for attempt in range(FRONTEND_BUILD_ATTEMPTS):
|
for attempt in range(FRONTEND_BUILD_ATTEMPTS):
|
||||||
|
self._checkpoint()
|
||||||
unparsed = self._build_with_frontend(root, cpg_path, scratch, language, exclude)
|
unparsed = self._build_with_frontend(root, cpg_path, scratch, language, exclude)
|
||||||
if unparsed is None:
|
if unparsed is None:
|
||||||
return best # the frontend is missing or failed outright; keep any earlier partial build
|
return best # the frontend is missing or failed outright; keep any earlier partial build
|
||||||
@@ -773,6 +821,8 @@ class JoernBackend:
|
|||||||
point -- "no rows" and "could not ask" look identical to a caller that only sees a dict, and a
|
point -- "no rows" and "could not ask" look identical to a caller that only sees a dict, and a
|
||||||
117k-line PHP CPG that threw while loading turned a failed scan into a clean bill of health.
|
117k-line PHP CPG that threw while loading turned a failed scan into a clean bill of health.
|
||||||
"""
|
"""
|
||||||
|
if not self._time_available():
|
||||||
|
return None
|
||||||
script = self.queries_dir / f"{query}.sc"
|
script = self.queries_dir / f"{query}.sc"
|
||||||
if not script.is_file():
|
if not script.is_file():
|
||||||
logger.warning("no such cpg query: %s", script)
|
logger.warning("no such cpg query: %s", script)
|
||||||
@@ -786,6 +836,8 @@ class JoernBackend:
|
|||||||
# {} -- and {} reads as "no rows found". The scan reported 500 regions examined and nothing found, in
|
# {} -- and {} reads as "no rows found". The scan reported 500 regions examined and nothing found, in
|
||||||
# 0.05 seconds of taint query, which is a quiet failure of exactly the kind the rung ladder cannot
|
# 0.05 seconds of taint query, which is a quiet failure of exactly the kind the rung ladder cannot
|
||||||
# catch because no verdict was ever produced to label.
|
# catch because no verdict was ever produced to label.
|
||||||
|
if not self._time_available():
|
||||||
|
return None
|
||||||
requests_file = cpg_path.parent / f"{query}-requests.json"
|
requests_file = cpg_path.parent / f"{query}-requests.json"
|
||||||
try:
|
try:
|
||||||
requests_file.write_text(payload)
|
requests_file.write_text(payload)
|
||||||
@@ -810,6 +862,8 @@ class JoernBackend:
|
|||||||
logger.warning("cpg batch %s failed: %s", query, detail)
|
logger.warning("cpg batch %s failed: %s", query, detail)
|
||||||
return None
|
return None
|
||||||
parsed = extract_payload(completed.stdout or "")
|
parsed = extract_payload(completed.stdout or "")
|
||||||
|
if not self._time_available():
|
||||||
|
return None
|
||||||
if not isinstance(parsed, Mapping):
|
if not isinstance(parsed, Mapping):
|
||||||
# Unparseable output is a failure, not an empty answer. A 117k-line PHP CPG that throws while
|
# Unparseable output is a failure, not an empty answer. A 117k-line PHP CPG that throws while
|
||||||
# loading printed a stack trace and no payload, and returning {} made the whole scan read as a
|
# loading printed a stack trace and no payload, and returning {} made the whole scan read as a
|
||||||
@@ -821,6 +875,8 @@ class JoernBackend:
|
|||||||
|
|
||||||
def query(self, cpg_path: Path, query: str, params: Mapping[str, object]) -> object | None:
|
def query(self, cpg_path: Path, query: str, params: Mapping[str, object]) -> object | None:
|
||||||
"""Run a shipped CPGQL script against a built CPG and return its fenced JSON payload."""
|
"""Run a shipped CPGQL script against a built CPG and return its fenced JSON payload."""
|
||||||
|
if not self._time_available():
|
||||||
|
return None
|
||||||
script = self.queries_dir / f"{query}.sc"
|
script = self.queries_dir / f"{query}.sc"
|
||||||
if not script.is_file():
|
if not script.is_file():
|
||||||
logger.warning("no such cpg query: %s", script)
|
logger.warning("no such cpg query: %s", script)
|
||||||
@@ -835,7 +891,96 @@ class JoernBackend:
|
|||||||
detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
|
detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
|
||||||
logger.warning("cpg query %s failed: %s", query, detail)
|
logger.warning("cpg query %s failed: %s", query, detail)
|
||||||
return None
|
return None
|
||||||
return extract_payload(completed.stdout or "")
|
parsed = extract_payload(completed.stdout or "")
|
||||||
|
return parsed if self._time_available() else None
|
||||||
|
|
||||||
|
def _note(self, reason: str) -> None:
|
||||||
|
if reason not in self._diagnostics:
|
||||||
|
self._diagnostics.append(reason)
|
||||||
|
self.last_failure = reason
|
||||||
|
|
||||||
|
def _time_available(self) -> bool:
|
||||||
|
if self._cancel_deadline is not None:
|
||||||
|
return False
|
||||||
|
if self.execution_budget is not None and time.monotonic() >= self.execution_budget.deadline_monotonic:
|
||||||
|
self._note("deadline_exhausted")
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _checkpoint(self) -> None:
|
||||||
|
if not self._time_available():
|
||||||
|
raise _DeadlineExpired
|
||||||
|
|
||||||
|
def _paths(self, root: Path, pattern: str): # type: ignore[no-untyped-def]
|
||||||
|
paths = root.rglob(pattern)
|
||||||
|
if self.execution_budget is None:
|
||||||
|
yield from sorted(paths)
|
||||||
|
else:
|
||||||
|
self._checkpoint()
|
||||||
|
for path in paths:
|
||||||
|
self._checkpoint()
|
||||||
|
yield path
|
||||||
|
|
||||||
|
def _cleanup_limit(self) -> float:
|
||||||
|
assert self.execution_budget is not None
|
||||||
|
ceiling = self.execution_budget.deadline_monotonic + self.execution_budget.cancellation_allowance_seconds
|
||||||
|
return min(ceiling, self._cancel_deadline) if self._cancel_deadline is not None else ceiling
|
||||||
|
|
||||||
|
def _cleanup(self, root: Path, *, final: bool = False) -> None:
|
||||||
|
if self.execution_budget is None:
|
||||||
|
shutil.rmtree(root, ignore_errors=True)
|
||||||
|
return
|
||||||
|
deadline = self._cleanup_limit() if final or self._cancel_deadline is not None else self.execution_budget.deadline_monotonic
|
||||||
|
try:
|
||||||
|
_remove_owned_tree(root, deadline)
|
||||||
|
except (OSError, _DeadlineExpired):
|
||||||
|
self._note("scratch_cleanup_incomplete")
|
||||||
|
|
||||||
|
def _run_bounded(self, command: list[str], *, timeout: float, cwd: Path | None) -> subprocess.CompletedProcess[str] | None:
|
||||||
|
assert self.execution_budget is not None
|
||||||
|
if not self._time_available():
|
||||||
|
return None
|
||||||
|
remaining = min(timeout, self.execution_budget.deadline_monotonic - time.monotonic())
|
||||||
|
if remaining <= 0:
|
||||||
|
self._note("deadline_exhausted")
|
||||||
|
return None
|
||||||
|
process = None
|
||||||
|
try:
|
||||||
|
if self.runner is not None:
|
||||||
|
result = self.runner(command, capture_output=True, text=True, timeout=remaining, check=False, cwd=str(cwd) if cwd else None)
|
||||||
|
else:
|
||||||
|
process = subprocess.Popen( # noqa: S603 -- internal argument array
|
||||||
|
command,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
cwd=str(cwd) if cwd else None,
|
||||||
|
env=self._jvm_env(),
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
remaining = min(remaining, max(0.0, self.execution_budget.deadline_monotonic - time.monotonic()))
|
||||||
|
stdout, stderr = process.communicate(timeout=remaining)
|
||||||
|
result = subprocess.CompletedProcess(command, process.returncode, stdout, stderr)
|
||||||
|
if not self._time_available():
|
||||||
|
return None
|
||||||
|
return result
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self._cancel_deadline = min(self._cleanup_limit(), time.monotonic() + self.execution_budget.cancellation_allowance_seconds)
|
||||||
|
self._note("deadline_exhausted" if time.monotonic() >= self.execution_budget.deadline_monotonic else "process_timeout")
|
||||||
|
return None
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
self._note("process_failed")
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
if process is not None:
|
||||||
|
# No Popen context manager: its implicit wait has no timeout. Always signal
|
||||||
|
# the original group ID, even if the launcher already exited, so descendants
|
||||||
|
# retaining pipes (or closing them) cannot survive it.
|
||||||
|
if not _terminate_bounded_group(process, self._cleanup_limit()):
|
||||||
|
self._note("process_cleanup_incomplete")
|
||||||
|
for stream in (process.stdout, process.stderr):
|
||||||
|
if stream is not None:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
def _heap_mb(self) -> int:
|
def _heap_mb(self) -> int:
|
||||||
if self.heap_mb > 0:
|
if self.heap_mb > 0:
|
||||||
@@ -874,6 +1019,8 @@ class JoernBackend:
|
|||||||
row it would OOM rather than merely degrade -- and it would look like the SCAN needing more memory,
|
row it would OOM rather than merely degrade -- and it would look like the SCAN needing more memory,
|
||||||
which is the wrong lesson entirely.
|
which is the wrong lesson entirely.
|
||||||
"""
|
"""
|
||||||
|
if self.execution_budget is not None:
|
||||||
|
return self._run_bounded(command, timeout=timeout, cwd=cwd)
|
||||||
run = self.runner if self.runner is not None else subprocess.run
|
run = self.runner if self.runner is not None else subprocess.run
|
||||||
if self.runner is not None: # tests inject a fake runner and never spawn anything
|
if self.runner is not None: # tests inject a fake runner and never spawn anything
|
||||||
try:
|
try:
|
||||||
@@ -900,6 +1047,64 @@ class JoernBackend:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _DeadlineExpired(Exception):
|
||||||
|
"""Internal cooperative boundary; public builds/queries retain failure semantics."""
|
||||||
|
|
||||||
|
|
||||||
|
def _terminate_bounded_group(process: subprocess.Popen[str], deadline: float) -> bool:
|
||||||
|
# SIGKILL immediately leaves the shared allowance for reaping and scratch cleanup.
|
||||||
|
# Unlike the legacy TERM/wait loop this also kills grandchildren after leader exit.
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGKILL)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
process.wait(timeout=max(0.0, deadline - time.monotonic()))
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_owned_tree(root: Path, deadline: float) -> None:
|
||||||
|
"""Bounded Linux dirfd cleanup, never following substituted directory links."""
|
||||||
|
|
||||||
|
def check() -> None:
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise _DeadlineExpired
|
||||||
|
|
||||||
|
def contents(fd: int) -> None:
|
||||||
|
with os.scandir(fd) as entries:
|
||||||
|
for entry in entries:
|
||||||
|
check()
|
||||||
|
info = os.stat(entry.name, dir_fd=fd, follow_symlinks=False)
|
||||||
|
if stat.S_ISDIR(info.st_mode):
|
||||||
|
child = os.open(entry.name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
||||||
|
try:
|
||||||
|
contents(child)
|
||||||
|
finally:
|
||||||
|
os.close(child)
|
||||||
|
check()
|
||||||
|
os.rmdir(entry.name, dir_fd=fd)
|
||||||
|
else:
|
||||||
|
os.unlink(entry.name, dir_fd=fd)
|
||||||
|
|
||||||
|
if not root.exists() and not root.is_symlink():
|
||||||
|
return
|
||||||
|
check()
|
||||||
|
if root.is_symlink():
|
||||||
|
root.unlink()
|
||||||
|
return
|
||||||
|
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||||
|
try:
|
||||||
|
contents(fd)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
check()
|
||||||
|
root.rmdir()
|
||||||
|
|
||||||
|
|
||||||
SCRATCH_PREFIX = "ousast-cpg-"
|
SCRATCH_PREFIX = "ousast-cpg-"
|
||||||
# How long an abandoned scratch tree may survive. Long enough that a scan running in another process is never
|
# How long an abandoned scratch tree may survive. Long enough that a scan running in another process is never
|
||||||
# disturbed, short enough that a machine scanning all day does not fill its disk.
|
# disturbed, short enough that a machine scanning all day does not fill its disk.
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ the region's risk score said beforehand.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
@@ -28,6 +29,7 @@ from typing import Any
|
|||||||
from ..cpg.backend import CpgResult
|
from ..cpg.backend import CpgResult
|
||||||
from .candidates import enumerate_candidates
|
from .candidates import enumerate_candidates
|
||||||
from .config_value import request_params as config_params
|
from .config_value import request_params as config_params
|
||||||
|
from .contracts import ExecutionBudget
|
||||||
from .dominance import request_params as dominance_params
|
from .dominance import request_params as dominance_params
|
||||||
from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows
|
from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows
|
||||||
from .ladder import Rung
|
from .ladder import Rung
|
||||||
@@ -127,6 +129,49 @@ def scan_repository(
|
|||||||
client: Any | None = None,
|
client: Any | None = None,
|
||||||
model: str = "",
|
model: str = "",
|
||||||
budget: ScanBudget | None = None,
|
budget: ScanBudget | None = None,
|
||||||
|
execution_budget: ExecutionBudget | None = None,
|
||||||
|
) -> ModelScanResult:
|
||||||
|
"""Run with an optional transaction deadline; graph ownership ends on every exit."""
|
||||||
|
started = time.monotonic()
|
||||||
|
owned: list[Any] = []
|
||||||
|
try:
|
||||||
|
result = _scan_repository_impl(
|
||||||
|
root,
|
||||||
|
regions,
|
||||||
|
backend=backend,
|
||||||
|
client=client if execution_budget is None else None,
|
||||||
|
model=model,
|
||||||
|
budget=budget,
|
||||||
|
execution_budget=execution_budget,
|
||||||
|
owned=owned,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
for cpg in owned:
|
||||||
|
_dispose(cpg)
|
||||||
|
diagnostics = list(result.degradations)
|
||||||
|
if execution_budget is not None:
|
||||||
|
if client is not None:
|
||||||
|
diagnostics.append({"stage": "model", "reason": "model_withheld_execution_budget"})
|
||||||
|
reasons = ["deadline_exhausted"] if time.monotonic() >= execution_budget.deadline_monotonic else []
|
||||||
|
for cpg in owned:
|
||||||
|
read_diagnostics = getattr(cpg, "execution_diagnostics", None)
|
||||||
|
if callable(read_diagnostics):
|
||||||
|
reasons.extend(read_diagnostics())
|
||||||
|
for reason in dict.fromkeys(reasons):
|
||||||
|
diagnostics.append({"stage": "model", "reason": reason})
|
||||||
|
return replace(result, degradations=tuple(diagnostics), seconds=round(time.monotonic() - started, 2))
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_repository_impl(
|
||||||
|
root: Path,
|
||||||
|
regions: Sequence[ScanRegion],
|
||||||
|
*,
|
||||||
|
backend: Any,
|
||||||
|
client: Any | None = None,
|
||||||
|
model: str = "",
|
||||||
|
budget: ScanBudget | None = None,
|
||||||
|
execution_budget: ExecutionBudget | None = None,
|
||||||
|
owned: list[Any],
|
||||||
) -> ModelScanResult:
|
) -> ModelScanResult:
|
||||||
"""Arbitrate a repository's regions under one CPG and one budget."""
|
"""Arbitrate a repository's regions under one CPG and one budget."""
|
||||||
limits = budget if budget is not None else ScanBudget()
|
limits = budget if budget is not None else ScanBudget()
|
||||||
@@ -140,7 +185,7 @@ def scan_repository(
|
|||||||
for region in regions:
|
for region in regions:
|
||||||
counts[region.language] = counts.get(region.language, 0) + 1
|
counts[region.language] = counts.get(region.language, 0) + 1
|
||||||
dominant = max(counts, key=lambda name: (counts[name], name)) if counts else ""
|
dominant = max(counts, key=lambda name: (counts[name], name)) if counts else ""
|
||||||
cpg = _build(backend, root, dominant)
|
cpg = _build(backend, root, dominant, execution_budget=execution_budget)
|
||||||
build_seconds = round(time.monotonic() - build_started, 2)
|
build_seconds = round(time.monotonic() - build_started, 2)
|
||||||
if cpg is None:
|
if cpg is None:
|
||||||
return ModelScanResult(
|
return ModelScanResult(
|
||||||
@@ -151,6 +196,8 @@ def scan_repository(
|
|||||||
degradations=({"stage": "model", "reason": "cpg_build_failed", "detail": getattr(backend, "last_failure", "")},),
|
degradations=({"stage": "model", "reason": "cpg_build_failed", "detail": getattr(backend, "last_failure", "")},),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
owned.append(cpg)
|
||||||
|
|
||||||
# A CPG the frontend built with files missing is not the repository the caller asked about, and no
|
# A CPG the frontend built with files missing is not the repository the caller asked about, and no
|
||||||
# verdict over it can say anything about the code that was dropped. There is no rung for that, because
|
# verdict over it can say anything about the code that was dropped. There is no rung for that, because
|
||||||
# the ladder labels verdicts and none was ever produced here -- so it is a degradation, named, with the
|
# the ladder labels verdicts and none was ever produced here -- so it is a degradation, named, with the
|
||||||
@@ -198,14 +245,14 @@ def scan_repository(
|
|||||||
tiers: list[tuple[str, str, str, int]] = []
|
tiers: list[tuple[str, str, str, int]] = []
|
||||||
per_kind: dict[str, float] = {}
|
per_kind: dict[str, float] = {}
|
||||||
evidence_by_pair: dict[tuple[str, str, str], Evidence] = {}
|
evidence_by_pair: dict[tuple[str, str, str], Evidence] = {}
|
||||||
if limits.tiering and callable(getattr(cpg, "run_batch", None)):
|
if _within_deadline(execution_budget) and limits.tiering and callable(getattr(cpg, "run_batch", None)):
|
||||||
candidates = (
|
evidence_candidates = (
|
||||||
ordered_regions[: (limits.evidence_regions or len(ordered_regions))]
|
ordered_regions[: (limits.evidence_regions or len(ordered_regions))]
|
||||||
if limits.order_by_evidence
|
if limits.order_by_evidence
|
||||||
else ordered_regions[: limits.max_regions]
|
else ordered_regions[: limits.max_regions]
|
||||||
)
|
)
|
||||||
tier_started = time.monotonic()
|
tier_started = time.monotonic()
|
||||||
evidence_by_pair, failed = _evidence_pass(cpg, _collect(candidates), hooks)
|
evidence_by_pair, failed = _evidence_pass(cpg, _collect(evidence_candidates), hooks)
|
||||||
per_kind["evidence"] = round(time.monotonic() - tier_started, 2)
|
per_kind["evidence"] = round(time.monotonic() - tier_started, 2)
|
||||||
if failed:
|
if failed:
|
||||||
degradations.append({"stage": "model", "reason": "query_failed", "kind": "evidence", "requests": failed})
|
degradations.append({"stage": "model", "reason": "query_failed", "kind": "evidence", "requests": failed})
|
||||||
@@ -227,6 +274,8 @@ def scan_repository(
|
|||||||
query_started = time.monotonic()
|
query_started = time.monotonic()
|
||||||
pruned = 0
|
pruned = 0
|
||||||
for kind, requests in _grouped(work, hook_callbacks=hooks).items():
|
for kind, requests in _grouped(work, hook_callbacks=hooks).items():
|
||||||
|
if not _within_deadline(execution_budget):
|
||||||
|
break
|
||||||
if kind == "taint" and limits.prune_tier0 and tier_by_rid:
|
if kind == "taint" and limits.prune_tier0 and tier_by_rid:
|
||||||
# Tier 0 is exact: a family with no sink in reach cannot yield a flow. Measured on a 637-file
|
# Tier 0 is exact: a family with no sink in reach cannot yield a flow. Measured on a 637-file
|
||||||
# plugin, that is 2,161 of 2,500 requests -- 86% of a query that timed out at forty minutes.
|
# plugin, that is 2,161 of 2,500 requests -- 86% of a query that timed out at forty minutes.
|
||||||
@@ -268,8 +317,15 @@ def scan_repository(
|
|||||||
rows_by_id.update(answered_batch)
|
rows_by_id.update(answered_batch)
|
||||||
else: # a backend without batching still works, one call at a time
|
else: # a backend without batching still works, one call at a time
|
||||||
for rid, params in requests.items():
|
for rid, params in requests.items():
|
||||||
|
if not _within_deadline(execution_budget):
|
||||||
|
break
|
||||||
answered = cpg.run(kind, params)
|
answered = cpg.run(kind, params)
|
||||||
rows_by_id[rid] = list(answered) if isinstance(answered, list) else []
|
if isinstance(answered, list):
|
||||||
|
rows_by_id[rid] = list(answered)
|
||||||
|
elif execution_budget is None:
|
||||||
|
rows_by_id[rid] = []
|
||||||
|
else:
|
||||||
|
degradations.append({"stage": "model", "reason": "query_failed", "kind": kind, "requests": 1})
|
||||||
per_kind[kind] = round(time.monotonic() - kind_started, 2)
|
per_kind[kind] = round(time.monotonic() - kind_started, 2)
|
||||||
query_seconds = round(time.monotonic() - query_started, 2)
|
query_seconds = round(time.monotonic() - query_started, 2)
|
||||||
|
|
||||||
@@ -287,6 +343,12 @@ def scan_repository(
|
|||||||
judged: set[tuple[str, str | None]] = set()
|
judged: set[tuple[str, str | None]] = set()
|
||||||
unasked: set[tuple[str, str | None]] = set()
|
unasked: set[tuple[str, str | None]] = set()
|
||||||
for rid, region, spec in work:
|
for rid, region, spec in work:
|
||||||
|
if not _within_deadline(execution_budget):
|
||||||
|
break
|
||||||
|
# Unanswered requests are deferred on the bounded path, never adjudicated
|
||||||
|
# from an invented empty response. Exact identities are added in task 3.2.
|
||||||
|
if execution_budget is not None and rid not in rows_by_id:
|
||||||
|
continue
|
||||||
exhausted = counted is not None and counted.calls >= limits.max_model_calls
|
exhausted = counted is not None and counted.calls >= limits.max_model_calls
|
||||||
family = getattr(spec, "family", "")
|
family = getattr(spec, "family", "")
|
||||||
prefetched = CpgResult(cpg_path=cpg.cpg_path, run=_prefetched(rows_by_id.get(rid, [])))
|
prefetched = CpgResult(cpg_path=cpg.cpg_path, run=_prefetched(rows_by_id.get(rid, [])))
|
||||||
@@ -331,7 +393,6 @@ def scan_repository(
|
|||||||
findings = _ordered(_deduplicated(collected))
|
findings = _ordered(_deduplicated(collected))
|
||||||
# The graph has answered everything it is going to. Its scratch tree holds the CPG, the request files and
|
# The graph has answered everything it is going to. Its scratch tree holds the CPG, the request files and
|
||||||
# joern's own working copy -- tens of megabytes per scan -- and nothing else reclaims it.
|
# joern's own working copy -- tens of megabytes per scan -- and nothing else reclaims it.
|
||||||
_dispose(cpg)
|
|
||||||
return ModelScanResult(
|
return ModelScanResult(
|
||||||
findings=findings,
|
findings=findings,
|
||||||
by_rung=_tally(findings),
|
by_rung=_tally(findings),
|
||||||
@@ -441,11 +502,26 @@ def _dispose(cpg: object) -> None:
|
|||||||
logger.warning("could not remove the cpg scratch directory: %s", exc)
|
logger.warning("could not remove the cpg scratch directory: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
def _build(backend: Any, root: Path, language: str) -> Any:
|
def _within_deadline(budget: ExecutionBudget | None) -> bool:
|
||||||
|
return budget is None or time.monotonic() < budget.deadline_monotonic
|
||||||
|
|
||||||
|
|
||||||
|
def _build(backend: Any, root: Path, language: str, *, execution_budget: ExecutionBudget | None = None) -> Any:
|
||||||
"""Build through the backend, passing the language and the vendored trees when the backend can use them."""
|
"""Build through the backend, passing the language and the vendored trees when the backend can use them."""
|
||||||
from .layout import vendored_directories
|
from .layout import vendored_directories
|
||||||
|
|
||||||
|
if not _within_deadline(execution_budget):
|
||||||
|
return None
|
||||||
excluded = vendored_directories(root) if root.is_dir() else ()
|
excluded = vendored_directories(root) if root.is_dir() else ()
|
||||||
|
if execution_budget is not None:
|
||||||
|
try:
|
||||||
|
inspect.signature(backend.build).bind(root, language=language, exclude=excluded, execution_budget=execution_budget)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
# An old backend cannot silently turn a bounded push into an ordinary scan.
|
||||||
|
return None
|
||||||
|
if not _within_deadline(execution_budget):
|
||||||
|
return None
|
||||||
|
return backend.build(root, language=language, exclude=excluded, execution_budget=execution_budget)
|
||||||
try:
|
try:
|
||||||
return backend.build(root, language=language, exclude=excluded)
|
return backend.build(root, language=language, exclude=excluded)
|
||||||
except TypeError: # a backend from before the retry existed, including every test double
|
except TypeError: # a backend from before the retry existed, including every test double
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ def test_the_seam_imports_nothing_outside_the_standard_library() -> None:
|
|||||||
"re",
|
"re",
|
||||||
"shutil",
|
"shutil",
|
||||||
"signal",
|
"signal",
|
||||||
|
"stat",
|
||||||
"subprocess",
|
"subprocess",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"time",
|
"time",
|
||||||
|
|||||||
@@ -0,0 +1,372 @@
|
|||||||
|
"""Absolute transaction deadlines on the real process boundary (pre-push 4.1/4.2)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from openultrasast.cpg.backend import BEGIN, END, JoernBackend
|
||||||
|
from openultrasast.model.contracts import ExecutionBudget
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_budget_launches_nothing(tmp_path: Path) -> None:
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append(command)
|
||||||
|
return subprocess.CompletedProcess(command, 0, "ok", "")
|
||||||
|
|
||||||
|
backend = JoernBackend(runner=run, execution_budget=ExecutionBudget(time.monotonic() - 1, 0.2))
|
||||||
|
assert backend._run(["unused"], timeout=300) is None
|
||||||
|
assert not calls
|
||||||
|
assert "deadline_exhausted" in backend.last_failure
|
||||||
|
|
||||||
|
|
||||||
|
def test_remaining_time_is_not_reset_between_commands() -> None:
|
||||||
|
remaining = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
remaining.append(kwargs["timeout"])
|
||||||
|
time.sleep(0.04)
|
||||||
|
return subprocess.CompletedProcess(command, 0, "", "")
|
||||||
|
|
||||||
|
backend = JoernBackend(runner=run, execution_budget=ExecutionBudget(time.monotonic() + 0.5, 0.2))
|
||||||
|
assert backend._run(["first"], timeout=300) is not None
|
||||||
|
assert backend._run(["second"], timeout=300) is not None
|
||||||
|
assert 0 < remaining[1] < remaining[0] <= 0.5
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("leader_exits", [False, True])
|
||||||
|
def test_hanging_process_group_killed_inside_shared_allowance(tmp_path: Path, leader_exits: bool) -> None:
|
||||||
|
source = tmp_path / "source.js"
|
||||||
|
source.write_bytes(b"const input = req.query.input;\n")
|
||||||
|
assert len(source.read_bytes()) == 31
|
||||||
|
pids = tmp_path / "pids"
|
||||||
|
script = """import os,signal,time,sys
|
||||||
|
signal.signal(signal.SIGTERM,signal.SIG_IGN)
|
||||||
|
child=os.fork()
|
||||||
|
if child==0:
|
||||||
|
while True: time.sleep(1)
|
||||||
|
open(sys.argv[1], 'w').write(str(os.getpid())+' '+str(child))
|
||||||
|
if sys.argv[2]=='yes': sys.exit(0)
|
||||||
|
while True: time.sleep(1)
|
||||||
|
"""
|
||||||
|
started = time.monotonic()
|
||||||
|
deadline = started + 0.25
|
||||||
|
backend = JoernBackend(execution_budget=ExecutionBudget(deadline, 0.3))
|
||||||
|
try:
|
||||||
|
assert backend._run([sys.executable, "-c", script, str(pids), "yes" if leader_exits else "no"], timeout=1) is None
|
||||||
|
assert time.monotonic() - started < 0.65
|
||||||
|
assert pids.is_file()
|
||||||
|
for pid in map(int, pids.read_text().split()):
|
||||||
|
status = Path(f"/proc/{pid}/stat")
|
||||||
|
for _ in range(30):
|
||||||
|
if not status.exists() or status.read_text().split()[2] == "Z":
|
||||||
|
break
|
||||||
|
time.sleep(0.005)
|
||||||
|
else:
|
||||||
|
pytest.fail(f"process {pid} survived cancellation")
|
||||||
|
finally:
|
||||||
|
if pids.exists():
|
||||||
|
for pid in map(int, pids.read_text().split()):
|
||||||
|
with contextlib.suppress(ProcessLookupError):
|
||||||
|
os.kill(pid, 9)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("hang_stage", ["probe", "frontend", "overlay", "census", "taint"])
|
||||||
|
def test_real_hanging_engine_stages_share_deadline_and_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, hang_stage: str) -> None:
|
||||||
|
from openultrasast.cpg import backend as module
|
||||||
|
|
||||||
|
root = tmp_path / "source"
|
||||||
|
root.mkdir()
|
||||||
|
source = root / "api.php"
|
||||||
|
source.write_bytes(b"<?php function f($x) { return $x; }\n")
|
||||||
|
assert len(source.read_bytes()) == 36
|
||||||
|
launcher = tmp_path / "engine"
|
||||||
|
event_log = tmp_path / "events"
|
||||||
|
pid_log = tmp_path / "pids"
|
||||||
|
launcher.write_text(f"""#!{sys.executable}
|
||||||
|
import os,sys,json,time,signal
|
||||||
|
from pathlib import Path
|
||||||
|
args=sys.argv[1:]
|
||||||
|
stage='probe' if '-r' in args else 'frontend' if '-o' in args else Path(args[args.index('--script')+1]).stem
|
||||||
|
with open({str(event_log)!r},'a') as f: f.write(stage+'\\n')
|
||||||
|
if stage=='probe':
|
||||||
|
data=Path(args[-1]).read_bytes()
|
||||||
|
print(len(data))
|
||||||
|
assert data
|
||||||
|
if stage=={hang_stage!r}:
|
||||||
|
signal.signal(signal.SIGTERM,signal.SIG_IGN)
|
||||||
|
child=os.fork()
|
||||||
|
if child==0:
|
||||||
|
while True: time.sleep(1)
|
||||||
|
Path({str(pid_log)!r}).write_text(str(os.getpid())+' '+str(child))
|
||||||
|
while True: time.sleep(1)
|
||||||
|
if stage=='frontend': Path(args[args.index('-o')+1]).write_bytes(b'controlled graph fixture')
|
||||||
|
if stage=='overlay':
|
||||||
|
graph=Path(next(a.split('=',1)[1] for a in args if a.startswith('cpgFile=')))
|
||||||
|
saved=Path.cwd()/'workspace'/graph.name/'cpg.bin'
|
||||||
|
saved.parent.mkdir(parents=True)
|
||||||
|
saved.write_bytes(b'controlled graph with overlays')
|
||||||
|
print({BEGIN!r})
|
||||||
|
print(json.dumps({{'files':'1','methods':'1','calls':'1','overlays':'dataflowOss'}} if stage!='taint' else {{'r':[]}}))
|
||||||
|
print({END!r})
|
||||||
|
""")
|
||||||
|
launcher.chmod(0o700)
|
||||||
|
monkeypatch.setattr(module.shutil, "which", lambda name: str(launcher))
|
||||||
|
monkeypatch.setattr(module, "joern_version", lambda: (4, 0, 625))
|
||||||
|
scratch = tmp_path / "scratch"
|
||||||
|
scratch.mkdir()
|
||||||
|
monkeypatch.setattr(module.tempfile, "mkdtemp", lambda **kwargs: str(scratch))
|
||||||
|
started = time.monotonic()
|
||||||
|
budget = ExecutionBudget(started + 0.55, 0.35)
|
||||||
|
backend = JoernBackend(execution_budget=budget)
|
||||||
|
graph = None
|
||||||
|
try:
|
||||||
|
graph = backend.build(root, language="php")
|
||||||
|
if hang_stage == "taint":
|
||||||
|
assert graph is not None
|
||||||
|
assert graph.run_batch is not None
|
||||||
|
assert graph.run_batch("taint", {"r": {}}) is None
|
||||||
|
assert graph.execution_diagnostics is not None
|
||||||
|
assert "deadline_exhausted" in graph.execution_diagnostics()
|
||||||
|
else:
|
||||||
|
assert graph is None
|
||||||
|
if graph is not None and graph.cleanup is not None:
|
||||||
|
graph.cleanup()
|
||||||
|
assert time.monotonic() - started < 1.0
|
||||||
|
assert event_log.read_text().splitlines()[-1] == hang_stage
|
||||||
|
assert not scratch.exists()
|
||||||
|
assert pid_log.exists(), "the controlled subprocess really reached the hanging stage"
|
||||||
|
for pid in map(int, pid_log.read_text().split()):
|
||||||
|
status = Path(f"/proc/{pid}/stat")
|
||||||
|
for _ in range(30):
|
||||||
|
if not status.exists() or status.read_text().split()[2] == "Z":
|
||||||
|
break
|
||||||
|
time.sleep(0.005)
|
||||||
|
else:
|
||||||
|
pytest.fail(f"engine child {pid} survived")
|
||||||
|
finally:
|
||||||
|
if pid_log.exists():
|
||||||
|
for pid in map(int, pid_log.read_text().split()):
|
||||||
|
with contextlib.suppress(ProcessLookupError):
|
||||||
|
os.kill(pid, 9)
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_build_budget_is_retained_without_mutating_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.cpg import backend as module
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append(kwargs["timeout"])
|
||||||
|
if "--output" in command:
|
||||||
|
Path(command[command.index("--output") + 1]).write_bytes(b"graph fixture")
|
||||||
|
return subprocess.CompletedProcess(command, 0, f'{BEGIN}\n{{"r":[]}}\n{END}', "")
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.shutil, "which", lambda name: name)
|
||||||
|
backend = JoernBackend(runner=run)
|
||||||
|
budget = ExecutionBudget(time.monotonic() + 1, 0.2)
|
||||||
|
graph = backend.build(tmp_path, language="javascript", execution_budget=budget)
|
||||||
|
assert graph is not None and graph.run_batch is not None and graph.cleanup is not None
|
||||||
|
assert backend.execution_budget is None
|
||||||
|
time.sleep(0.02)
|
||||||
|
assert graph.run_batch("taint", {"r": {}}) == {"r": []}
|
||||||
|
assert 0 < calls[1] < calls[0] <= 1
|
||||||
|
graph.cleanup()
|
||||||
|
assert not graph.cpg_path.parent.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_early_process_timeout_stops_fallback_and_retries(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.cpg import backend as module
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append(command)
|
||||||
|
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.shutil, "which", lambda name: name)
|
||||||
|
backend = JoernBackend(runner=run, execution_budget=ExecutionBudget(time.monotonic() + 100, 0.2))
|
||||||
|
assert backend.build(tmp_path, language="javascript") is None
|
||||||
|
assert len(calls) == 1
|
||||||
|
assert "process_timeout" in backend._diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_timeout_consumes_same_absolute_budget(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.cpg import backend as module
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append(kwargs["timeout"])
|
||||||
|
Path(command[command.index("-o") + 1]).write_bytes(b"partial graph")
|
||||||
|
time.sleep(0.04)
|
||||||
|
return subprocess.CompletedProcess(command, 0, "WARN Failed to process 'x.php'", "")
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.shutil, "which", lambda name: name)
|
||||||
|
backend = JoernBackend(runner=run, execution_budget=ExecutionBudget(time.monotonic() + 1, 0.2))
|
||||||
|
assert backend._build_with_retries_raw(tmp_path, tmp_path / "cpg.bin", tmp_path, "php") == ("x.php",)
|
||||||
|
assert len(calls) == 4
|
||||||
|
assert all(a > b > 0 for a, b in zip(calls, calls[1:], strict=False))
|
||||||
|
assert calls[0] <= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_shard_failure_cannot_become_complete_empty_answer(tmp_path: Path) -> None:
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append(command)
|
||||||
|
if len(calls) == 2:
|
||||||
|
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
|
||||||
|
return subprocess.CompletedProcess(command, 0, f'{BEGIN}\n{{"r":[]}}\n{END}', "")
|
||||||
|
|
||||||
|
backend = JoernBackend(runner=run, execution_budget=ExecutionBudget(time.monotonic() + 1, 0.2))
|
||||||
|
assert backend.query_batch_across([tmp_path / "a.bin", tmp_path / "b.bin", tmp_path / "c.bin"], "taint", {"r": {}}) is None
|
||||||
|
assert len(calls) == 2
|
||||||
|
assert "shard_query_incomplete" in backend._diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_cannot_reset_expired_cancellation_allowance(tmp_path: Path) -> None:
|
||||||
|
graph = tmp_path / "owned"
|
||||||
|
graph.mkdir()
|
||||||
|
(graph / "data").write_bytes(b"data")
|
||||||
|
backend = JoernBackend(execution_budget=ExecutionBudget(time.monotonic() - 1, 0.1))
|
||||||
|
started = time.monotonic()
|
||||||
|
backend._cleanup(graph, final=True)
|
||||||
|
assert time.monotonic() - started < 0.05
|
||||||
|
assert graph.exists()
|
||||||
|
assert "scratch_cleanup_incomplete" in backend._diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_does_not_follow_substituted_directory_links(tmp_path: Path) -> None:
|
||||||
|
outside = tmp_path / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
(outside / "keep").write_bytes(b"keep")
|
||||||
|
scratch = tmp_path / "scratch"
|
||||||
|
scratch.mkdir()
|
||||||
|
(scratch / "link").symlink_to(outside, target_is_directory=True)
|
||||||
|
backend = JoernBackend(execution_budget=ExecutionBudget(time.monotonic() + 1, 0.2))
|
||||||
|
backend._cleanup(scratch, final=True)
|
||||||
|
assert not scratch.exists()
|
||||||
|
assert (outside / "keep").read_bytes() == b"keep"
|
||||||
|
|
||||||
|
|
||||||
|
def test_driver_propagates_budget_and_rejects_legacy_without_launch(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.cpg.backend import CpgResult
|
||||||
|
from openultrasast.model.scan import scan_repository
|
||||||
|
|
||||||
|
budget = ExecutionBudget(time.monotonic() + 1, 0.2)
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
seen = None
|
||||||
|
cleaned = False
|
||||||
|
|
||||||
|
def build(self, root, *, language, exclude, execution_budget):
|
||||||
|
self.seen = execution_budget
|
||||||
|
return CpgResult(tmp_path / "graph", lambda *args: [], cleanup=self.cleanup)
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
self.cleaned = True
|
||||||
|
|
||||||
|
backend = Backend()
|
||||||
|
result = scan_repository(tmp_path, [], backend=backend, execution_budget=budget, client=object())
|
||||||
|
assert backend.seen is budget and backend.cleaned
|
||||||
|
assert any(d["reason"] == "model_withheld_execution_budget" for d in result.degradations)
|
||||||
|
|
||||||
|
class Legacy:
|
||||||
|
called = False
|
||||||
|
|
||||||
|
def build(self, root):
|
||||||
|
self.called = True
|
||||||
|
raise AssertionError("must not launch without a deadline")
|
||||||
|
|
||||||
|
legacy = Legacy()
|
||||||
|
result = scan_repository(tmp_path, [], backend=legacy, execution_budget=budget)
|
||||||
|
assert not legacy.called
|
||||||
|
assert any(d["reason"] == "cpg_build_failed" for d in result.degradations)
|
||||||
|
|
||||||
|
|
||||||
|
def test_budgeted_driver_does_not_retry_internal_typeerror(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.model.scan import scan_repository
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def build(self, root, **kwargs):
|
||||||
|
self.calls += 1
|
||||||
|
raise TypeError("internal backend bug")
|
||||||
|
|
||||||
|
backend = Backend()
|
||||||
|
with pytest.raises(TypeError, match="internal backend bug"):
|
||||||
|
scan_repository(tmp_path, [], backend=backend, execution_budget=ExecutionBudget(time.monotonic() + 1, 0.2))
|
||||||
|
assert backend.calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_cancelled_graph_session_does_not_poison_next_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.cpg import backend as module
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append(command)
|
||||||
|
if len(calls) == 1:
|
||||||
|
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
|
||||||
|
if "--output" in command:
|
||||||
|
Path(command[command.index("--output") + 1]).write_bytes(b"graph")
|
||||||
|
return subprocess.CompletedProcess(command, 0, f'{BEGIN}\n{{"r":[]}}\n{END}', "")
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.shutil, "which", lambda name: name)
|
||||||
|
backend = JoernBackend(runner=run)
|
||||||
|
budget = ExecutionBudget(time.monotonic() + 2, 0.2)
|
||||||
|
assert backend.build(tmp_path, execution_budget=budget) is None
|
||||||
|
assert backend._cancel_deadline is None
|
||||||
|
graph = backend.build(tmp_path, execution_budget=budget)
|
||||||
|
assert graph is not None and graph.cleanup is not None
|
||||||
|
assert graph.execution_diagnostics is not None
|
||||||
|
assert graph.execution_diagnostics() == ()
|
||||||
|
graph.cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
def test_driver_cleans_graph_on_unexpected_exception(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.cpg.backend import CpgResult
|
||||||
|
from openultrasast.model import scan
|
||||||
|
|
||||||
|
cleaned = []
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
def build(self, root, **kwargs):
|
||||||
|
return CpgResult(tmp_path / "graph", lambda *args: [], cleanup=lambda: cleaned.append(True))
|
||||||
|
|
||||||
|
def broken(*args, **kwargs):
|
||||||
|
raise RuntimeError("broken context")
|
||||||
|
|
||||||
|
monkeypatch.setattr(scan, "_hook_callbacks", broken)
|
||||||
|
with pytest.raises(RuntimeError, match="broken context"):
|
||||||
|
scan.scan_repository(tmp_path, [], backend=Backend(), execution_budget=ExecutionBudget(time.monotonic() + 1, 0.2))
|
||||||
|
assert cleaned == [True]
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_nonbatch_query_is_unjudged_on_budgeted_path(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.cpg.backend import CpgResult
|
||||||
|
from openultrasast.model.regions import ScanRegion
|
||||||
|
from openultrasast.model.scan import scan_repository
|
||||||
|
|
||||||
|
source = tmp_path / "api.py"
|
||||||
|
source.write_bytes(b"def run(x): return x\n")
|
||||||
|
assert len(source.read_bytes()) > 0
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
def build(self, root, **kwargs):
|
||||||
|
return CpgResult(tmp_path / "graph", lambda *args: None)
|
||||||
|
|
||||||
|
region = ScanRegion(path="api.py", function="run", language="python", families=("injection",), rank=1.0, source="entry_point")
|
||||||
|
result = scan_repository(tmp_path, [region], backend=Backend(), execution_budget=ExecutionBudget(time.monotonic() + 1, 0.2))
|
||||||
|
assert result.regions_scanned == 0 and result.regions_unjudged == 1
|
||||||
|
assert any(d["reason"] == "query_failed" for d in result.degradations)
|
||||||
Reference in New Issue
Block a user