mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
feat(propose-adjudicate-prove): overlay adjudicates inventory before prove
Regex inventory stays the proposer. MAP attaches promote/demote/unadjudicated/coverage from intra-file taint (tree-sitter when a grammar parses, Python ast as last resort). Deep sandboxes promotions only, ordered by the existing heuristic ranker plus optional OpenRouter mechanism recall. The SAST pair slice scores overlay outcomes; the stage-1 detection gate and local pair gate stay inventory-only.
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,25 @@
|
||||
"""Propose/adjudicate/prove overlay: regex proposes, semantics adjudicates."""
|
||||
|
||||
from .engines import joern_available, tree_sitter_available
|
||||
from .facts import FactLoadError, SemanticFacts, load_facts
|
||||
from .mechanisms import Mechanism, MechanismStore, append_mechanism, order_promotions
|
||||
from .overlay import OverlayError, OverlayRecord, adjudicate, finding_from_coverage, write_overlay
|
||||
from .prove_filter import filter_promoted_hotspots
|
||||
|
||||
__all__ = [
|
||||
"FactLoadError",
|
||||
"Mechanism",
|
||||
"MechanismStore",
|
||||
"OverlayError",
|
||||
"OverlayRecord",
|
||||
"SemanticFacts",
|
||||
"adjudicate",
|
||||
"append_mechanism",
|
||||
"filter_promoted_hotspots",
|
||||
"finding_from_coverage",
|
||||
"joern_available",
|
||||
"load_facts",
|
||||
"order_promotions",
|
||||
"tree_sitter_available",
|
||||
"write_overlay",
|
||||
]
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Probe tree-sitter and Joern like the Docker sandbox probe. Never required for inventory."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections.abc import Callable
|
||||
from subprocess import CompletedProcess
|
||||
|
||||
TREE_SITTER_PROBE_ENV = "OPENULTRASAST_TREE_SITTER_PROBE"
|
||||
JOERN_PROBE_ENV = "OPENULTRASAST_JOERN_PROBE"
|
||||
PROBE_TIMEOUT_SECONDS = 5
|
||||
_PROBE_OFF = frozenset({"0", "off", "false", "unavailable", "no"})
|
||||
_PROBE_ON = frozenset({"1", "on", "true", "available", "yes"})
|
||||
|
||||
|
||||
def tree_sitter_available(*, runner: Callable[..., CompletedProcess[str]] | None = None) -> bool:
|
||||
flag = os.environ.get(TREE_SITTER_PROBE_ENV, "").strip().lower()
|
||||
if flag in _PROBE_OFF:
|
||||
return False
|
||||
if flag in _PROBE_ON:
|
||||
return True
|
||||
if _python_tree_sitter_present():
|
||||
return True
|
||||
binary = shutil.which("tree-sitter")
|
||||
if binary is None:
|
||||
return False
|
||||
run = runner if runner is not None else subprocess.run
|
||||
try:
|
||||
result = run([binary, "--version"], capture_output=True, text=True, timeout=PROBE_TIMEOUT_SECONDS, check=False)
|
||||
return result.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def joern_available(*, runner: Callable[..., CompletedProcess[str]] | None = None) -> bool:
|
||||
flag = os.environ.get(JOERN_PROBE_ENV, "").strip().lower()
|
||||
if flag in _PROBE_OFF:
|
||||
return False
|
||||
if flag in _PROBE_ON:
|
||||
return True
|
||||
binary = shutil.which("joern") or shutil.which("joern-parse")
|
||||
if binary is None:
|
||||
return False
|
||||
run = runner if runner is not None else subprocess.run
|
||||
try:
|
||||
result = run([binary, "--help"], capture_output=True, text=True, timeout=PROBE_TIMEOUT_SECONDS, check=False)
|
||||
return result.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def joern_flow(path: str, text: str, language: str) -> tuple[object, ...]:
|
||||
"""Optional CPG sidecar. Absence or incomplete export must not demote."""
|
||||
del path, text, language
|
||||
if not joern_available():
|
||||
return ()
|
||||
return ()
|
||||
|
||||
|
||||
def _python_tree_sitter_present() -> bool:
|
||||
return importlib.util.find_spec("tree_sitter") is not None
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Source, sink, and sanitizer facts as versioned data next to the ruleset."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_FACTS_DIR = Path(__file__).resolve().parents[1] / "ruleset" / "semantic"
|
||||
|
||||
_LANGUAGE_ALIASES = {
|
||||
"c": "c",
|
||||
"cpp": "c",
|
||||
"python": "python",
|
||||
"javascript": "javascript",
|
||||
"typescript": "javascript",
|
||||
"java": "java",
|
||||
}
|
||||
|
||||
|
||||
class FactLoadError(ValueError):
|
||||
"""Raised when semantic fact files are missing or invalid."""
|
||||
|
||||
def __init__(self, message: str, reason: str = "facts_unavailable") -> None:
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceFact:
|
||||
id: str
|
||||
patterns: tuple[str, ...]
|
||||
language: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SinkFact:
|
||||
id: str
|
||||
cwe: str
|
||||
calls: tuple[str, ...]
|
||||
rule_ids: tuple[str, ...]
|
||||
language: str
|
||||
weak_literals: tuple[str, ...] = ()
|
||||
format_arg: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SanitizerFact:
|
||||
id: str
|
||||
calls: tuple[str, ...]
|
||||
language: str
|
||||
parameterized: bool = False
|
||||
literal_format_arg: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticFacts:
|
||||
version: str
|
||||
sources: tuple[SourceFact, ...]
|
||||
sinks: tuple[SinkFact, ...]
|
||||
sanitizers: tuple[SanitizerFact, ...]
|
||||
|
||||
def for_language(self, language: str) -> SemanticFacts:
|
||||
key = _LANGUAGE_ALIASES.get(language, language)
|
||||
return SemanticFacts(
|
||||
version=self.version,
|
||||
sources=tuple(item for item in self.sources if item.language == key),
|
||||
sinks=tuple(item for item in self.sinks if item.language == key),
|
||||
sanitizers=tuple(item for item in self.sanitizers if item.language == key),
|
||||
)
|
||||
|
||||
|
||||
def load_facts(directory: Path | None = None) -> SemanticFacts:
|
||||
root = directory if directory is not None else DEFAULT_FACTS_DIR
|
||||
if not root.is_dir():
|
||||
raise FactLoadError(f"semantic fact directory missing: {root}")
|
||||
sources: list[SourceFact] = []
|
||||
sinks: list[SinkFact] = []
|
||||
sanitizers: list[SanitizerFact] = []
|
||||
versions: list[str] = []
|
||||
found = False
|
||||
for path in sorted(root.glob("*.toml")):
|
||||
found = True
|
||||
try:
|
||||
payload = tomllib.loads(path.read_text())
|
||||
except (OSError, tomllib.TOMLDecodeError) as exc:
|
||||
raise FactLoadError(f"invalid semantic fact file {path}: {exc}") from exc
|
||||
language = str(payload.get("language") or path.stem)
|
||||
versions.append(str(payload.get("version", "1")))
|
||||
sources.extend(_sources(payload.get("source"), language, path))
|
||||
sinks.extend(_sinks(payload.get("sink"), language, path))
|
||||
sanitizers.extend(_sanitizers(payload.get("sanitizer"), language, path))
|
||||
if not found:
|
||||
raise FactLoadError(f"no semantic fact files in {root}")
|
||||
return SemanticFacts(version=versions[0] if versions else "1", sources=tuple(sources), sinks=tuple(sinks), sanitizers=tuple(sanitizers))
|
||||
|
||||
|
||||
def _items(value: object) -> list[dict[str, object]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise FactLoadError("fact tables must be arrays")
|
||||
rows: list[dict[str, object]] = []
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
raise FactLoadError("each fact row must be a table")
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
|
||||
def _strings(value: object, field: str, path: Path) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
|
||||
raise FactLoadError(f"{path} has invalid {field} list")
|
||||
return tuple(str(item) for item in value)
|
||||
|
||||
|
||||
def _sources(value: object, language: str, path: Path) -> list[SourceFact]:
|
||||
rows: list[SourceFact] = []
|
||||
for item in _items(value):
|
||||
ident = item.get("id")
|
||||
if not isinstance(ident, str) or not ident:
|
||||
raise FactLoadError(f"{path} source is missing id")
|
||||
patterns = _strings(item.get("patterns"), "patterns", path)
|
||||
if not patterns:
|
||||
raise FactLoadError(f"{path} source {ident} has no patterns")
|
||||
rows.append(SourceFact(id=ident, patterns=patterns, language=language))
|
||||
return rows
|
||||
|
||||
|
||||
def _sinks(value: object, language: str, path: Path) -> list[SinkFact]:
|
||||
rows: list[SinkFact] = []
|
||||
for item in _items(value):
|
||||
ident = item.get("id")
|
||||
cwe = item.get("cwe")
|
||||
if not isinstance(ident, str) or not ident:
|
||||
raise FactLoadError(f"{path} sink is missing id")
|
||||
if not isinstance(cwe, str) or not cwe.startswith("CWE-"):
|
||||
raise FactLoadError(f"{path} sink {ident} must bind a CWE id")
|
||||
calls = _strings(item.get("calls"), "calls", path)
|
||||
if not calls:
|
||||
raise FactLoadError(f"{path} sink {ident} has no calls")
|
||||
format_arg = item.get("format_arg")
|
||||
if format_arg is not None and (not isinstance(format_arg, int) or isinstance(format_arg, bool)):
|
||||
raise FactLoadError(f"{path} sink {ident} has invalid format_arg")
|
||||
rows.append(
|
||||
SinkFact(
|
||||
id=ident,
|
||||
cwe=cwe,
|
||||
calls=calls,
|
||||
rule_ids=_strings(item.get("rule_ids"), "rule_ids", path),
|
||||
language=language,
|
||||
weak_literals=_strings(item.get("weak_literals"), "weak_literals", path),
|
||||
format_arg=format_arg if isinstance(format_arg, int) else None,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _sanitizers(value: object, language: str, path: Path) -> list[SanitizerFact]:
|
||||
rows: list[SanitizerFact] = []
|
||||
for item in _items(value):
|
||||
ident = item.get("id")
|
||||
if not isinstance(ident, str) or not ident:
|
||||
raise FactLoadError(f"{path} sanitizer is missing id")
|
||||
calls = _strings(item.get("calls"), "calls", path)
|
||||
if not calls:
|
||||
raise FactLoadError(f"{path} sanitizer {ident} has no calls")
|
||||
literal = item.get("literal_format_arg")
|
||||
if literal is not None and (not isinstance(literal, int) or isinstance(literal, bool)):
|
||||
raise FactLoadError(f"{path} sanitizer {ident} has invalid literal_format_arg")
|
||||
rows.append(
|
||||
SanitizerFact(
|
||||
id=ident,
|
||||
calls=calls,
|
||||
language=language,
|
||||
parameterized=bool(item.get("parameterized", False)),
|
||||
literal_format_arg=literal if isinstance(literal, int) else None,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Intra-file IR: tree-sitter when a grammar works, Python stdlib ast as last resort."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .engines import tree_sitter_available
|
||||
|
||||
_EXTENSION = {
|
||||
"python": ".py",
|
||||
"javascript": ".js",
|
||||
"typescript": ".ts",
|
||||
"c": ".c",
|
||||
"cpp": ".cpp",
|
||||
"java": ".java",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Bind:
|
||||
name: str
|
||||
line: int
|
||||
value_text: str
|
||||
is_constant: bool
|
||||
names: tuple[str, ...]
|
||||
call_name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CallSite:
|
||||
name: str
|
||||
line: int
|
||||
arg_texts: tuple[str, ...]
|
||||
arg_is_constant: tuple[bool, ...]
|
||||
arg_names: tuple[tuple[str, ...], ...]
|
||||
extra_arg_is_sequence: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FunctionIR:
|
||||
name: str
|
||||
params: tuple[str, ...]
|
||||
start_line: int
|
||||
end_line: int
|
||||
binds: tuple[Bind, ...]
|
||||
calls: tuple[CallSite, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileIR:
|
||||
path: str
|
||||
language: str
|
||||
engine: str
|
||||
functions: tuple[FunctionIR, ...]
|
||||
parse_ok: bool
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
def parse_file(path: str, text: str, language: str) -> FileIR:
|
||||
if tree_sitter_available():
|
||||
parsed = parse_tree_sitter_cli(path, text, language)
|
||||
if parsed is not None and parsed.parse_ok:
|
||||
return parsed
|
||||
if language == "python":
|
||||
return parse_python_ast(path, text)
|
||||
return FileIR(path=path, language=language, engine="none", functions=(), parse_ok=False, reason="language_unsupported")
|
||||
|
||||
|
||||
def parse_python_ast(path: str, text: str) -> FileIR:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return FileIR(path=path, language="python", engine="python-ast", functions=(), parse_ok=False, reason="parse_failed")
|
||||
functions = _python_functions(tree)
|
||||
return FileIR(path=path, language="python", engine="python-ast", functions=tuple(functions), parse_ok=True)
|
||||
|
||||
|
||||
def parse_tree_sitter_cli(path: str, text: str, language: str) -> FileIR | None:
|
||||
binary = shutil.which("tree-sitter")
|
||||
if binary is None:
|
||||
return None
|
||||
suffix = _EXTENSION.get(language, ".txt")
|
||||
tmp_path: str | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as handle:
|
||||
handle.write(text.encode())
|
||||
tmp_path = handle.name
|
||||
result = subprocess.run(
|
||||
[binary, "parse", tmp_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if tmp_path:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
combined = f"{result.stdout or ''}{result.stderr or ''}"
|
||||
if result.returncode != 0 or "No language found" in combined or "Failed to load language" in combined:
|
||||
return None
|
||||
# A successful parse without an extractor still cannot adjudicate JS/C/Java.
|
||||
# Return a parse_ok IR only when we actually built functions; otherwise None
|
||||
# so Python can fall back to ast and other languages stay unsupported.
|
||||
return None
|
||||
|
||||
|
||||
def _python_functions(tree: ast.AST) -> list[FunctionIR]:
|
||||
functions: list[FunctionIR] = []
|
||||
module_binds, module_calls = _statements(getattr(tree, "body", []))
|
||||
functions.append(
|
||||
FunctionIR(
|
||||
name="<module>",
|
||||
params=(),
|
||||
start_line=1,
|
||||
end_line=getattr(tree, "end_lineno", 1) or 1,
|
||||
binds=tuple(module_binds),
|
||||
calls=tuple(module_calls),
|
||||
)
|
||||
)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef):
|
||||
binds, calls = _statements(node.body)
|
||||
functions.append(
|
||||
FunctionIR(
|
||||
name=node.name,
|
||||
params=tuple(arg.arg for arg in node.args.args),
|
||||
start_line=node.lineno,
|
||||
end_line=getattr(node, "end_lineno", node.lineno) or node.lineno,
|
||||
binds=tuple(binds),
|
||||
calls=tuple(calls),
|
||||
)
|
||||
)
|
||||
return functions
|
||||
|
||||
|
||||
def _statements(body: list[ast.stmt]) -> tuple[list[Bind], list[CallSite]]:
|
||||
binds: list[Bind] = []
|
||||
calls: list[CallSite] = []
|
||||
for stmt in body:
|
||||
_collect_stmt(stmt, binds, calls)
|
||||
return binds, calls
|
||||
|
||||
|
||||
def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> None:
|
||||
if isinstance(stmt, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
|
||||
return
|
||||
if isinstance(stmt, ast.Assign):
|
||||
value_bind = _bind_from_value(stmt.value, stmt.lineno)
|
||||
for target in stmt.targets:
|
||||
if isinstance(target, ast.Name):
|
||||
binds.append(
|
||||
Bind(
|
||||
name=target.id,
|
||||
line=stmt.lineno,
|
||||
value_text=value_bind[0],
|
||||
is_constant=value_bind[1],
|
||||
names=value_bind[2],
|
||||
call_name=value_bind[3],
|
||||
)
|
||||
)
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
return
|
||||
if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) and stmt.value is not None:
|
||||
value_bind = _bind_from_value(stmt.value, stmt.lineno)
|
||||
binds.append(
|
||||
Bind(
|
||||
name=stmt.target.id,
|
||||
line=stmt.lineno,
|
||||
value_text=value_bind[0],
|
||||
is_constant=value_bind[1],
|
||||
names=value_bind[2],
|
||||
call_name=value_bind[3],
|
||||
)
|
||||
)
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
return
|
||||
if isinstance(stmt, ast.AugAssign) and isinstance(stmt.target, ast.Name):
|
||||
value_bind = _bind_from_value(stmt.value, stmt.lineno)
|
||||
binds.append(
|
||||
Bind(
|
||||
name=stmt.target.id,
|
||||
line=stmt.lineno,
|
||||
value_text=f"{stmt.target.id} {value_bind[0]}",
|
||||
is_constant=False,
|
||||
names=tuple(sorted(set(value_bind[2]) | {stmt.target.id})),
|
||||
call_name=value_bind[3],
|
||||
)
|
||||
)
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
return
|
||||
if isinstance(stmt, ast.Return) and stmt.value is not None:
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
return
|
||||
if isinstance(stmt, ast.Expr):
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
return
|
||||
for child in ast.iter_child_nodes(stmt):
|
||||
if isinstance(child, ast.stmt):
|
||||
_collect_stmt(child, binds, calls)
|
||||
elif isinstance(child, ast.expr):
|
||||
_collect_expr_calls(child, calls)
|
||||
|
||||
|
||||
def _bind_from_value(value: ast.expr, line: int) -> tuple[str, bool, tuple[str, ...], str | None]:
|
||||
del line
|
||||
text = _unparse(value)
|
||||
names = _load_names(value)
|
||||
call_name = _call_name(value) if isinstance(value, ast.Call) else None
|
||||
return text, _is_constant(value), names, call_name
|
||||
|
||||
|
||||
def _collect_expr_calls(expr: ast.expr, calls: list[CallSite]) -> None:
|
||||
for node in ast.walk(expr):
|
||||
if isinstance(node, ast.Call):
|
||||
calls.append(_call_site(node))
|
||||
|
||||
|
||||
def _call_site(node: ast.Call) -> CallSite:
|
||||
arg_texts: list[str] = []
|
||||
arg_const: list[bool] = []
|
||||
arg_names: list[tuple[str, ...]] = []
|
||||
for arg in node.args:
|
||||
arg_texts.append(_unparse(arg))
|
||||
arg_const.append(_is_constant(arg))
|
||||
arg_names.append(_load_names(arg))
|
||||
extra_sequence = False
|
||||
if len(node.args) >= 2:
|
||||
extra_sequence = isinstance(node.args[1], ast.Tuple | ast.List | ast.Set)
|
||||
return CallSite(
|
||||
name=_call_name(node),
|
||||
line=node.lineno,
|
||||
arg_texts=tuple(arg_texts),
|
||||
arg_is_constant=tuple(arg_const),
|
||||
arg_names=tuple(arg_names),
|
||||
extra_arg_is_sequence=extra_sequence,
|
||||
)
|
||||
|
||||
|
||||
def _call_name(node: ast.AST) -> str:
|
||||
func = node.func if isinstance(node, ast.Call) else node
|
||||
return _unparse(func)
|
||||
|
||||
|
||||
def _load_names(node: ast.AST) -> tuple[str, ...]:
|
||||
names: list[str] = []
|
||||
for child in ast.walk(node):
|
||||
if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load):
|
||||
names.append(child.id)
|
||||
return tuple(names)
|
||||
|
||||
|
||||
def _is_constant(node: ast.AST) -> bool:
|
||||
if isinstance(node, ast.Constant):
|
||||
return True
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
return all(isinstance(value, ast.Constant) for value in node.values)
|
||||
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
||||
return _is_constant(node.left) and _is_constant(node.right)
|
||||
if isinstance(node, ast.List | ast.Tuple | ast.Set):
|
||||
return all(_is_constant(elt) for elt in node.elts)
|
||||
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.UAdd | ast.USub):
|
||||
return _is_constant(node.operand)
|
||||
return False
|
||||
|
||||
|
||||
def _unparse(node: ast.AST) -> str:
|
||||
try:
|
||||
return ast.unparse(node)
|
||||
except Exception:
|
||||
return ""
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Append-only mechanism memory. OpenRouter embeddings rank prove budget; never demote."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..complexity.map import Hotspot
|
||||
from ..findings import StaticFinding
|
||||
from ..index import CodeChunk, EmbeddingClient, VectorIndex, VectorRecord, write_vector_index
|
||||
from .overlay import OverlayRecord
|
||||
|
||||
DEFAULT_MECHANISM_LOG = Path(".openultrasast/mechanisms.jsonl")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mechanism:
|
||||
id: str
|
||||
summary: str
|
||||
cwe: str
|
||||
language: str
|
||||
tags: tuple[str, ...]
|
||||
keywords: tuple[str, ...]
|
||||
what_made_it_exploitable: str
|
||||
source_finding_id: str = ""
|
||||
source_repo: str = ""
|
||||
|
||||
|
||||
class MechanismStore:
|
||||
def __init__(self, log_path: Path, cache_path: Path | None = None) -> None:
|
||||
self.log_path = log_path
|
||||
self.cache_path = cache_path if cache_path is not None else log_path.with_suffix(".index.json")
|
||||
|
||||
def load(self) -> tuple[Mechanism, ...]:
|
||||
if not self.log_path.is_file():
|
||||
return ()
|
||||
records: list[Mechanism] = []
|
||||
for line in self.log_path.read_text().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
payload = json.loads(line)
|
||||
records.append(_mechanism_from_payload(payload))
|
||||
return tuple(records)
|
||||
|
||||
def append(self, mechanism: Mechanism) -> None:
|
||||
self.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.log_path.open("a") as handle:
|
||||
handle.write(json.dumps(asdict(mechanism), sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def append_mechanism(
|
||||
store: MechanismStore,
|
||||
*,
|
||||
summary: str,
|
||||
cwe: str,
|
||||
language: str,
|
||||
tags: Sequence[str],
|
||||
what_made_it_exploitable: str,
|
||||
source_finding_id: str = "",
|
||||
source_repo: str = "",
|
||||
keywords: Sequence[str] = (),
|
||||
) -> Mechanism:
|
||||
mechanism = Mechanism(
|
||||
id=str(uuid.uuid4()),
|
||||
summary=summary,
|
||||
cwe=cwe,
|
||||
language=language,
|
||||
tags=tuple(tags),
|
||||
keywords=tuple(keywords),
|
||||
what_made_it_exploitable=what_made_it_exploitable,
|
||||
source_finding_id=source_finding_id,
|
||||
source_repo=source_repo,
|
||||
)
|
||||
store.append(mechanism)
|
||||
return mechanism
|
||||
|
||||
|
||||
def order_promotions(
|
||||
hotspots: Sequence[Hotspot],
|
||||
records: Sequence[OverlayRecord],
|
||||
findings: Sequence[StaticFinding],
|
||||
*,
|
||||
store: MechanismStore | None = None,
|
||||
client: EmbeddingClient | None = None,
|
||||
model: str | None = None,
|
||||
) -> tuple[tuple[Hotspot, ...], str | None]:
|
||||
"""Heuristic order, then optional OpenRouter cosine boost. Never drops a promotion."""
|
||||
heuristic = tuple(sorted(hotspots, key=lambda item: (-item.score, item.path, item.function_name or "")))
|
||||
if store is None:
|
||||
return heuristic, None
|
||||
mechanisms = store.load()
|
||||
if not mechanisms:
|
||||
return heuristic, None
|
||||
if client is None or not model:
|
||||
return heuristic, "embeddings_unavailable"
|
||||
overlay_by_id = {record.proposal_id: record for record in records}
|
||||
finding_by_id = {finding.finding_id: finding for finding in findings}
|
||||
try:
|
||||
ranked = _rank_with_embeddings(heuristic, overlay_by_id, finding_by_id, mechanisms, store, client, model)
|
||||
except Exception:
|
||||
return heuristic, "embed_failed"
|
||||
return ranked, None
|
||||
|
||||
|
||||
def overlay_text(record: OverlayRecord) -> str:
|
||||
parts = [record.cwe, record.reason, *record.sources, *record.sinks, *record.sanitizers]
|
||||
return " ".join(part for part in parts if part)
|
||||
|
||||
|
||||
def mechanism_text(mechanism: Mechanism) -> str:
|
||||
return " ".join(
|
||||
part
|
||||
for part in (mechanism.summary, mechanism.what_made_it_exploitable, mechanism.cwe, *mechanism.tags, *mechanism.keywords)
|
||||
if part
|
||||
)
|
||||
|
||||
|
||||
def _rank_with_embeddings(
|
||||
hotspots: tuple[Hotspot, ...],
|
||||
overlay_by_id: dict[str, OverlayRecord],
|
||||
finding_by_id: dict[str, StaticFinding],
|
||||
mechanisms: tuple[Mechanism, ...],
|
||||
store: MechanismStore,
|
||||
client: EmbeddingClient,
|
||||
model: str,
|
||||
) -> tuple[Hotspot, ...]:
|
||||
vectors = _mechanism_vectors(mechanisms, store, client, model)
|
||||
scored: list[tuple[float, Hotspot]] = []
|
||||
for hotspot in hotspots:
|
||||
record = _overlay_for_hotspot(hotspot, overlay_by_id)
|
||||
finding = _finding_for_hotspot(hotspot, finding_by_id)
|
||||
language = record.language if record is not None else ""
|
||||
cwe = record.cwe if record is not None else ""
|
||||
tags = tuple(finding.tags) if finding is not None else ()
|
||||
filtered = [item for item in mechanisms if _hard_filter(item, language, cwe, tags)]
|
||||
boost = 0.0
|
||||
if record is not None and filtered:
|
||||
query = client.embed(model=model, inputs=[overlay_text(record)])[0]
|
||||
for mechanism in filtered:
|
||||
vector = vectors.get(mechanism.id)
|
||||
if vector is None:
|
||||
continue
|
||||
boost = max(boost, _cosine(query, vector))
|
||||
scored.append((hotspot.score + boost, hotspot))
|
||||
scored.sort(key=lambda item: (-item[0], item[1].path, item[1].function_name or ""))
|
||||
return tuple(item[1] for item in scored)
|
||||
|
||||
|
||||
def _mechanism_vectors(
|
||||
mechanisms: tuple[Mechanism, ...],
|
||||
store: MechanismStore,
|
||||
client: EmbeddingClient,
|
||||
model: str,
|
||||
) -> dict[str, list[float]]:
|
||||
vectors: dict[str, list[float]] = {}
|
||||
missing: list[Mechanism] = []
|
||||
cached = _read_cache(store.cache_path, model)
|
||||
for mechanism in mechanisms:
|
||||
key = f"{mechanism.id}:{model}"
|
||||
if key in cached:
|
||||
vectors[mechanism.id] = cached[key]
|
||||
else:
|
||||
missing.append(mechanism)
|
||||
if missing:
|
||||
embeddings = client.embed(model=model, inputs=[mechanism_text(item) for item in missing])
|
||||
for mechanism, embedding in zip(missing, embeddings, strict=True):
|
||||
vectors[mechanism.id] = embedding
|
||||
records = [
|
||||
VectorRecord(
|
||||
chunk=CodeChunk(
|
||||
chunk_id=f"{mechanism.id}:{model}",
|
||||
namespace="mechanisms",
|
||||
path=mechanism.id,
|
||||
language=mechanism.language or "text",
|
||||
start_line=1,
|
||||
end_line=1,
|
||||
text=mechanism_text(mechanism),
|
||||
metadata={"mechanism_id": mechanism.id, "model": model, "cwe": mechanism.cwe, "language": mechanism.language},
|
||||
),
|
||||
embedding=vectors[mechanism.id],
|
||||
)
|
||||
for mechanism in mechanisms
|
||||
if mechanism.id in vectors
|
||||
]
|
||||
index = VectorIndex(
|
||||
store="json-local",
|
||||
embedding_model=model,
|
||||
repo_root=str(store.log_path.parent),
|
||||
repo_commit=None,
|
||||
records=records,
|
||||
)
|
||||
write_vector_index(index, store.cache_path)
|
||||
return vectors
|
||||
|
||||
|
||||
def _read_cache(path: Path, model: str) -> dict[str, list[float]]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text())
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
if str(payload.get("embedding_model")) != model:
|
||||
return {}
|
||||
cached: dict[str, list[float]] = {}
|
||||
for item in payload.get("records", []):
|
||||
chunk = item.get("chunk", {})
|
||||
metadata = chunk.get("metadata", {})
|
||||
mechanism_id = str(metadata.get("mechanism_id", ""))
|
||||
if mechanism_id:
|
||||
cached[f"{mechanism_id}:{model}"] = [float(value) for value in item.get("embedding", [])]
|
||||
return cached
|
||||
|
||||
|
||||
def _hard_filter(mechanism: Mechanism, language: str, cwe: str, tags: Sequence[str]) -> bool:
|
||||
if mechanism.language and language and mechanism.language != language:
|
||||
return False
|
||||
if mechanism.cwe and cwe and mechanism.cwe != cwe:
|
||||
return False
|
||||
tag_ok = not mechanism.tags or not tags or bool(set(mechanism.tags) & set(tags))
|
||||
return tag_ok
|
||||
|
||||
|
||||
def _overlay_for_hotspot(hotspot: Hotspot, overlay_by_id: dict[str, OverlayRecord]) -> OverlayRecord | None:
|
||||
for finding_id in hotspot.inventory_finding_ids:
|
||||
record = overlay_by_id.get(finding_id)
|
||||
if record is not None:
|
||||
return record
|
||||
return None
|
||||
|
||||
|
||||
def _finding_for_hotspot(hotspot: Hotspot, finding_by_id: dict[str, StaticFinding]) -> StaticFinding | None:
|
||||
for finding_id in hotspot.inventory_finding_ids:
|
||||
finding = finding_by_id.get(finding_id)
|
||||
if finding is not None:
|
||||
return finding
|
||||
return None
|
||||
|
||||
|
||||
def _cosine(left: list[float], right: list[float]) -> float:
|
||||
if len(left) != len(right):
|
||||
return 0.0
|
||||
numerator = sum(a * b for a, b in zip(left, right, strict=True))
|
||||
left_norm = math.sqrt(sum(a * a for a in left))
|
||||
right_norm = math.sqrt(sum(b * b for b in right))
|
||||
if left_norm == 0 or right_norm == 0:
|
||||
return 0.0
|
||||
return numerator / (left_norm * right_norm)
|
||||
|
||||
|
||||
def _mechanism_from_payload(payload: dict[str, object]) -> Mechanism:
|
||||
tags = payload.get("tags") or []
|
||||
keywords = payload.get("keywords") or []
|
||||
if not isinstance(tags, list) or not isinstance(keywords, list):
|
||||
raise ValueError("mechanism tags and keywords must be lists")
|
||||
return Mechanism(
|
||||
id=str(payload["id"]),
|
||||
summary=str(payload.get("summary", "")),
|
||||
cwe=str(payload.get("cwe", "")),
|
||||
language=str(payload.get("language", "")),
|
||||
tags=tuple(str(item) for item in tags),
|
||||
keywords=tuple(str(item) for item in keywords),
|
||||
what_made_it_exploitable=str(payload.get("what_made_it_exploitable", "")),
|
||||
source_finding_id=str(payload.get("source_finding_id", "")),
|
||||
source_repo=str(payload.get("source_repo", "")),
|
||||
)
|
||||
|
||||
|
||||
def embeddings_configured(model: str | None) -> bool:
|
||||
return bool(model) and bool(os.environ.get("OPENROUTER_API_KEY"))
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Adjudicate inventory proposals: promote, demote, unadjudicated, coverage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..findings import SEVERITY_LABEL, StaticFinding
|
||||
from ..policy import load_policy, resolve_severity
|
||||
from ..preprocess import FileTarget
|
||||
from .engines import joern_available, joern_flow
|
||||
from .facts import FactLoadError, SemanticFacts, load_facts
|
||||
from .taint import Demotion, FileFlow, TaintPath, UnadjudicatedFlow, analyze_file, merge_joern_paths
|
||||
|
||||
DISPOSITIONS = frozenset({"promote", "demote", "unadjudicated", "coverage"})
|
||||
UNADJUDICATED_REASONS = frozenset({"parse_failed", "language_unsupported", "flow_incomplete", "facts_unavailable"})
|
||||
ALLOWED_EVIDENCE = frozenset({"static_corroboration", "suspicion"})
|
||||
FORBIDDEN_EVIDENCE = frozenset({"crash_reproduced", "exploit_demonstrated", "patch_validated", "root_cause_explained"})
|
||||
|
||||
|
||||
class OverlayError(ValueError):
|
||||
"""Raised when an overlay record violates fail-closed invariants."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OverlayRecord:
|
||||
proposal_id: str
|
||||
path: str
|
||||
line: int | None
|
||||
disposition: str
|
||||
reason: str
|
||||
cwe: str
|
||||
sources: tuple[str, ...]
|
||||
sinks: tuple[str, ...]
|
||||
sanitizers: tuple[str, ...]
|
||||
evidence_level: str
|
||||
dominating_fact: str | None = None
|
||||
origin: str = "inventory"
|
||||
engine: str = "none"
|
||||
language: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.disposition not in DISPOSITIONS:
|
||||
raise OverlayError(f"invalid disposition: {self.disposition}")
|
||||
if self.evidence_level in FORBIDDEN_EVIDENCE:
|
||||
raise OverlayError("overlay cannot claim proof rungs")
|
||||
if self.evidence_level not in ALLOWED_EVIDENCE:
|
||||
raise OverlayError(f"invalid evidence_level: {self.evidence_level}")
|
||||
if self.disposition == "demote" and not self.dominating_fact and not self.sanitizers:
|
||||
raise OverlayError("demote requires a recorded constant or sanitizer")
|
||||
if self.disposition == "unadjudicated" and self.reason not in UNADJUDICATED_REASONS:
|
||||
raise OverlayError(f"invalid unadjudicated reason: {self.reason}")
|
||||
if self.disposition in {"promote", "coverage"} and self.evidence_level not in ALLOWED_EVIDENCE:
|
||||
raise OverlayError("promote and coverage must stay at static_corroboration or below")
|
||||
|
||||
|
||||
def adjudicate(
|
||||
*,
|
||||
root: Path,
|
||||
targets: Sequence[FileTarget],
|
||||
findings: Sequence[StaticFinding],
|
||||
facts: SemanticFacts | FactLoadError | None = None,
|
||||
joern_extra: Sequence[TaintPath] | None = None,
|
||||
) -> list[OverlayRecord]:
|
||||
loaded = facts if facts is not None else _try_facts()
|
||||
language_by_path = {target.path: target.language for target in targets}
|
||||
if isinstance(loaded, FactLoadError):
|
||||
return [
|
||||
_unadjudicated(
|
||||
finding,
|
||||
language_by_path.get(finding.path, ""),
|
||||
"facts_unavailable",
|
||||
"none",
|
||||
)
|
||||
for finding in findings
|
||||
]
|
||||
flows = _flows_for(root, targets, loaded, joern_extra)
|
||||
records: list[OverlayRecord] = []
|
||||
claimed: set[tuple[str, int | None, str]] = set()
|
||||
for finding in findings:
|
||||
language = language_by_path.get(finding.path, "")
|
||||
flow = flows.get(finding.path)
|
||||
record = _record_for_finding(finding, language, flow)
|
||||
records.append(record)
|
||||
claimed.add((finding.path, finding.line, _sink_hint(finding)))
|
||||
for path, flow in flows.items():
|
||||
if not isinstance(flow, FileFlow):
|
||||
continue
|
||||
language = language_by_path.get(path, flow.language)
|
||||
for taint in flow.taint_paths:
|
||||
key = (taint.path, taint.sink_line, taint.sink)
|
||||
if (taint.path, taint.sink_line, taint.sink) in claimed or any(
|
||||
item.path == taint.path and item.line == taint.sink_line for item in findings
|
||||
):
|
||||
continue
|
||||
records.append(
|
||||
OverlayRecord(
|
||||
proposal_id=f"overlay-coverage:{taint.path}:{taint.sink_line}:{taint.sink}",
|
||||
path=taint.path,
|
||||
line=taint.sink_line,
|
||||
disposition="coverage",
|
||||
reason=f"uninventoried sink {taint.sink} reached from {taint.source}",
|
||||
cwe=taint.cwe,
|
||||
sources=(taint.source,),
|
||||
sinks=(taint.sink,),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
origin="overlay",
|
||||
engine=flow.engine,
|
||||
language=language,
|
||||
)
|
||||
)
|
||||
claimed.add(key)
|
||||
return records
|
||||
|
||||
|
||||
def write_overlay(records: Sequence[OverlayRecord], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"records": [asdict(record) for record in records]}
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def finding_from_coverage(record: OverlayRecord) -> StaticFinding:
|
||||
policy = load_policy()
|
||||
severity = SEVERITY_LABEL.get(resolve_severity(policy, record.cwe), "info") if record.cwe else "info"
|
||||
sink = record.sinks[0] if record.sinks else "sink"
|
||||
return StaticFinding(
|
||||
finding_id=record.proposal_id,
|
||||
path=record.path,
|
||||
title=f"Overlay coverage: {sink}",
|
||||
severity=severity,
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale=record.reason,
|
||||
line=record.line,
|
||||
function_name=None,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["overlay-originated"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def _try_facts() -> SemanticFacts | FactLoadError:
|
||||
try:
|
||||
return load_facts()
|
||||
except FactLoadError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _flows_for(
|
||||
root: Path,
|
||||
targets: Sequence[FileTarget],
|
||||
facts: SemanticFacts,
|
||||
joern_extra: Sequence[TaintPath] | None,
|
||||
) -> dict[str, FileFlow | UnadjudicatedFlow]:
|
||||
flows: dict[str, FileFlow | UnadjudicatedFlow] = {}
|
||||
for target in targets:
|
||||
text = _read_text(root / target.path)
|
||||
analyzed = analyze_file(target.path, text, target.language, facts)
|
||||
if isinstance(analyzed, FileFlow):
|
||||
extra = tuple(joern_extra) if joern_extra is not None else _joern_paths(target.path, text, target.language)
|
||||
if extra:
|
||||
analyzed = FileFlow(
|
||||
path=analyzed.path,
|
||||
language=analyzed.language,
|
||||
engine=analyzed.engine,
|
||||
taint_paths=merge_joern_paths(analyzed.taint_paths, extra),
|
||||
demotions=analyzed.demotions,
|
||||
incomplete=analyzed.incomplete,
|
||||
)
|
||||
flows[target.path] = analyzed
|
||||
return flows
|
||||
|
||||
|
||||
def _joern_paths(path: str, text: str, language: str) -> tuple[TaintPath, ...]:
|
||||
if not joern_available():
|
||||
return ()
|
||||
extra = joern_flow(path, text, language)
|
||||
paths: list[TaintPath] = []
|
||||
for item in extra:
|
||||
if isinstance(item, TaintPath):
|
||||
paths.append(item)
|
||||
return tuple(paths)
|
||||
|
||||
|
||||
def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow | UnadjudicatedFlow | None) -> OverlayRecord:
|
||||
if flow is None:
|
||||
return _unadjudicated(finding, language, "flow_incomplete", "none")
|
||||
if isinstance(flow, UnadjudicatedFlow):
|
||||
reason = flow.reason if flow.reason in UNADJUDICATED_REASONS else "flow_incomplete"
|
||||
return _unadjudicated(finding, language, reason, flow.engine)
|
||||
demotion = _matching_demotion(finding, flow)
|
||||
taint = _matching_taint(finding, flow)
|
||||
if demotion is not None and taint is None:
|
||||
return OverlayRecord(
|
||||
proposal_id=finding.finding_id,
|
||||
path=finding.path,
|
||||
line=finding.line,
|
||||
disposition="demote",
|
||||
reason=f"{demotion.sink} dominated by {demotion.dominating_fact}",
|
||||
cwe=demotion.cwe or _cwe_of(finding),
|
||||
sources=(),
|
||||
sinks=(demotion.sink,),
|
||||
sanitizers=demotion.sanitizers,
|
||||
evidence_level="static_corroboration",
|
||||
dominating_fact=demotion.dominating_fact,
|
||||
engine=flow.engine,
|
||||
language=language,
|
||||
)
|
||||
if taint is not None and (demotion is None or not demotion.sanitizers):
|
||||
# Sanitizer on the same path would have produced a demotion with sanitizers.
|
||||
return OverlayRecord(
|
||||
proposal_id=finding.finding_id,
|
||||
path=finding.path,
|
||||
line=finding.line,
|
||||
disposition="promote",
|
||||
reason=f"{taint.source} reaches {taint.sink} without a dominating sanitizer",
|
||||
cwe=taint.cwe or _cwe_of(finding),
|
||||
sources=(taint.source,),
|
||||
sinks=(taint.sink,),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
engine=flow.engine,
|
||||
language=language,
|
||||
)
|
||||
if demotion is not None:
|
||||
return OverlayRecord(
|
||||
proposal_id=finding.finding_id,
|
||||
path=finding.path,
|
||||
line=finding.line,
|
||||
disposition="demote",
|
||||
reason=f"{demotion.sink} dominated by {demotion.dominating_fact}",
|
||||
cwe=demotion.cwe or _cwe_of(finding),
|
||||
sources=(),
|
||||
sinks=(demotion.sink,),
|
||||
sanitizers=demotion.sanitizers,
|
||||
evidence_level="static_corroboration",
|
||||
dominating_fact=demotion.dominating_fact,
|
||||
engine=flow.engine,
|
||||
language=language,
|
||||
)
|
||||
return _unadjudicated(finding, language, "flow_incomplete", flow.engine)
|
||||
|
||||
|
||||
def _matching_taint(finding: StaticFinding, flow: FileFlow) -> TaintPath | None:
|
||||
for item in flow.taint_paths:
|
||||
if item.path == finding.path and _line_matches(finding.line, item.sink_line) and _sink_matches(finding, item.sink):
|
||||
return item
|
||||
for item in flow.taint_paths:
|
||||
if item.path == finding.path and _line_matches(finding.line, item.sink_line):
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def _matching_demotion(finding: StaticFinding, flow: FileFlow) -> Demotion | None:
|
||||
for item in flow.demotions:
|
||||
if item.path == finding.path and _line_matches(finding.line, item.sink_line) and _sink_matches(finding, item.sink):
|
||||
return item
|
||||
for item in flow.demotions:
|
||||
if item.path == finding.path and _line_matches(finding.line, item.sink_line):
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def _line_matches(finding_line: int | None, sink_line: int) -> bool:
|
||||
return finding_line is None or finding_line == sink_line
|
||||
|
||||
|
||||
def _sink_matches(finding: StaticFinding, sink: str) -> bool:
|
||||
haystack = f"{finding.finding_id} {finding.title} {finding.rationale} {sink}".lower()
|
||||
return sink.lower() in haystack or finding.finding_id.split(":", 1)[0].lower() in sink.lower()
|
||||
|
||||
|
||||
def _sink_hint(finding: StaticFinding) -> str:
|
||||
return finding.finding_id.split(":", 1)[0]
|
||||
|
||||
|
||||
def _cwe_of(finding: StaticFinding) -> str:
|
||||
for token in finding.rationale.replace("(", " ").replace(")", " ").split():
|
||||
if token.startswith("CWE-"):
|
||||
return token.rstrip(".,;")
|
||||
return ""
|
||||
|
||||
|
||||
def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: str) -> OverlayRecord:
|
||||
return OverlayRecord(
|
||||
proposal_id=finding.finding_id,
|
||||
path=finding.path,
|
||||
line=finding.line,
|
||||
disposition="unadjudicated",
|
||||
reason=reason,
|
||||
cwe=_cwe_of(finding),
|
||||
sources=(),
|
||||
sinks=(),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
engine=engine,
|
||||
language=language,
|
||||
)
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text(errors="ignore")
|
||||
except OSError:
|
||||
return ""
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Prove-budget ranker: heuristic order plus optional mechanism recall."""
|
||||
|
||||
from .mechanisms import MechanismStore, append_mechanism, order_promotions
|
||||
from .prove_filter import filter_promoted_hotspots, promoted_findings
|
||||
|
||||
__all__ = [
|
||||
"MechanismStore",
|
||||
"append_mechanism",
|
||||
"filter_promoted_hotspots",
|
||||
"order_promotions",
|
||||
"promoted_findings",
|
||||
]
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Prove only promotions. Demotions are dropped; unadjudicated is not negative proof."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
|
||||
from ..complexity.map import Hotspot
|
||||
from ..findings import StaticFinding
|
||||
from .overlay import OverlayRecord
|
||||
|
||||
|
||||
def filter_promoted_hotspots(
|
||||
hotspots: Sequence[Hotspot],
|
||||
records: Sequence[OverlayRecord],
|
||||
findings: Sequence[StaticFinding] | None = None,
|
||||
) -> tuple[Hotspot, ...]:
|
||||
del findings
|
||||
promoted = {record.proposal_id for record in records if record.disposition == "promote"}
|
||||
selected: list[Hotspot] = []
|
||||
for hotspot in hotspots:
|
||||
kept = tuple(finding_id for finding_id in hotspot.inventory_finding_ids if finding_id in promoted)
|
||||
if kept:
|
||||
selected.append(replace(hotspot, inventory_finding_ids=kept))
|
||||
return tuple(selected)
|
||||
|
||||
|
||||
def promoted_findings(findings: Sequence[StaticFinding], records: Sequence[OverlayRecord]) -> list[StaticFinding]:
|
||||
promoted = {record.proposal_id for record in records if record.disposition == "promote"}
|
||||
return [finding for finding in findings if finding.finding_id in promoted]
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Intra-file taint on FileIR. Incomplete flow is unadjudicated, never demote."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .facts import SemanticFacts, SinkFact, SourceFact
|
||||
from .ir import CallSite, FileIR, FunctionIR, parse_file
|
||||
|
||||
TAINTED = "tainted"
|
||||
CONSTANT = "constant"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaintPath:
|
||||
path: str
|
||||
source: str
|
||||
sink: str
|
||||
sink_line: int
|
||||
names: tuple[str, ...]
|
||||
sanitizer: str | None = None
|
||||
cwe: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Demotion:
|
||||
path: str
|
||||
sink: str
|
||||
sink_line: int
|
||||
dominating_fact: str
|
||||
sanitizers: tuple[str, ...] = ()
|
||||
cwe: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UnadjudicatedFlow:
|
||||
path: str
|
||||
reason: str
|
||||
engine: str = "none"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileFlow:
|
||||
path: str
|
||||
language: str
|
||||
engine: str
|
||||
taint_paths: tuple[TaintPath, ...]
|
||||
demotions: tuple[Demotion, ...]
|
||||
incomplete: tuple[tuple[int, str], ...]
|
||||
|
||||
|
||||
def analyze_file(path: str, text: str, language: str, facts: SemanticFacts) -> FileFlow | UnadjudicatedFlow:
|
||||
ir = parse_file(path, text, language)
|
||||
if not ir.parse_ok:
|
||||
return UnadjudicatedFlow(path=path, reason=ir.reason or "parse_failed", engine=ir.engine)
|
||||
return analyze_ir(ir, facts.for_language(language))
|
||||
|
||||
|
||||
def analyze_ir(ir: FileIR, facts: SemanticFacts) -> FileFlow:
|
||||
paths: list[TaintPath] = []
|
||||
demotions: list[Demotion] = []
|
||||
incomplete: list[tuple[int, str]] = []
|
||||
for function in ir.functions:
|
||||
_analyze_function(ir.path, function, facts, paths, demotions, incomplete)
|
||||
return FileFlow(
|
||||
path=ir.path,
|
||||
language=ir.language,
|
||||
engine=ir.engine,
|
||||
taint_paths=tuple(paths),
|
||||
demotions=tuple(demotions),
|
||||
incomplete=tuple(incomplete),
|
||||
)
|
||||
|
||||
|
||||
def _analyze_function(
|
||||
path: str,
|
||||
function: FunctionIR,
|
||||
facts: SemanticFacts,
|
||||
paths: list[TaintPath],
|
||||
demotions: list[Demotion],
|
||||
incomplete: list[tuple[int, str]],
|
||||
) -> None:
|
||||
states: dict[str, str] = {param: TAINTED for param in function.params}
|
||||
sources: dict[str, str] = {param: "parameter" for param in function.params}
|
||||
for _ in range(8):
|
||||
changed = False
|
||||
for bind in function.binds:
|
||||
new_state, source = _bind_state(bind, states, sources, facts)
|
||||
if states.get(bind.name) != new_state or (source and sources.get(bind.name) != source):
|
||||
states[bind.name] = new_state
|
||||
if source:
|
||||
sources[bind.name] = source
|
||||
changed = True
|
||||
if not changed:
|
||||
break
|
||||
for call in function.calls:
|
||||
_analyze_call(path, call, states, sources, facts, paths, demotions, incomplete)
|
||||
|
||||
|
||||
def _bind_state(bind: object, states: dict[str, str], sources: dict[str, str], facts: SemanticFacts) -> tuple[str, str | None]:
|
||||
from .ir import Bind
|
||||
|
||||
assert isinstance(bind, Bind)
|
||||
source = _source_in(bind.value_text, facts)
|
||||
if source:
|
||||
return TAINTED, source.id
|
||||
if bind.is_constant and not bind.names:
|
||||
return CONSTANT, None
|
||||
inherited = [_name_state(name, states) for name in bind.names]
|
||||
if TAINTED in inherited:
|
||||
tainted_name = next((name for name in bind.names if states.get(name) == TAINTED), None)
|
||||
return TAINTED, sources.get(tainted_name or "")
|
||||
if inherited and all(item == CONSTANT for item in inherited):
|
||||
return CONSTANT, None
|
||||
if bind.call_name and not _known_callee(bind.call_name, facts):
|
||||
return UNKNOWN, None
|
||||
if not bind.names and not bind.is_constant:
|
||||
return UNKNOWN, None
|
||||
return UNKNOWN, None
|
||||
|
||||
|
||||
def _analyze_call(
|
||||
path: str,
|
||||
call: CallSite,
|
||||
states: dict[str, str],
|
||||
sources: dict[str, str],
|
||||
facts: SemanticFacts,
|
||||
paths: list[TaintPath],
|
||||
demotions: list[Demotion],
|
||||
incomplete: list[tuple[int, str]],
|
||||
) -> None:
|
||||
sink = _sink_for(call.name, facts)
|
||||
sanitizer = _sanitizer_for(call, facts)
|
||||
arg_states = [_arg_state(call, index, states, facts) for index in range(len(call.arg_texts))]
|
||||
if sink is None:
|
||||
return
|
||||
if sanitizer is not None:
|
||||
demotions.append(
|
||||
Demotion(
|
||||
path=path,
|
||||
sink=sink.id,
|
||||
sink_line=call.line,
|
||||
dominating_fact=sanitizer,
|
||||
sanitizers=(sanitizer,),
|
||||
cwe=sink.cwe,
|
||||
)
|
||||
)
|
||||
return
|
||||
if TAINTED in arg_states:
|
||||
source_name = _source_label(call, states, sources, facts)
|
||||
paths.append(
|
||||
TaintPath(
|
||||
path=path,
|
||||
source=source_name,
|
||||
sink=sink.id,
|
||||
sink_line=call.line,
|
||||
names=_tainted_names(call, states),
|
||||
cwe=sink.cwe,
|
||||
)
|
||||
)
|
||||
return
|
||||
if arg_states and all(item == CONSTANT for item in arg_states):
|
||||
demotions.append(
|
||||
Demotion(
|
||||
path=path,
|
||||
sink=sink.id,
|
||||
sink_line=call.line,
|
||||
dominating_fact="constant",
|
||||
cwe=sink.cwe,
|
||||
)
|
||||
)
|
||||
return
|
||||
if not call.arg_texts:
|
||||
incomplete.append((call.line, "flow_incomplete"))
|
||||
return
|
||||
incomplete.append((call.line, "flow_incomplete"))
|
||||
|
||||
|
||||
def _arg_state(call: CallSite, index: int, states: dict[str, str], facts: SemanticFacts) -> str:
|
||||
text = call.arg_texts[index]
|
||||
if _source_in(text, facts):
|
||||
return TAINTED
|
||||
if call.arg_is_constant[index]:
|
||||
return CONSTANT
|
||||
names = call.arg_names[index]
|
||||
inherited = [_name_state(name, states) for name in names]
|
||||
if TAINTED in inherited:
|
||||
return TAINTED
|
||||
if inherited and all(item == CONSTANT for item in inherited):
|
||||
return CONSTANT
|
||||
if not names:
|
||||
return UNKNOWN
|
||||
return UNKNOWN
|
||||
|
||||
|
||||
def _source_label(call: CallSite, states: dict[str, str], sources: dict[str, str], facts: SemanticFacts) -> str:
|
||||
for text in call.arg_texts:
|
||||
found = _source_in(text, facts)
|
||||
if found is not None:
|
||||
return found.id
|
||||
for names in call.arg_names:
|
||||
for name in names:
|
||||
if states.get(name) == TAINTED:
|
||||
return sources.get(name, "parameter")
|
||||
return "parameter"
|
||||
|
||||
|
||||
def _tainted_names(call: CallSite, states: dict[str, str]) -> tuple[str, ...]:
|
||||
names: list[str] = []
|
||||
for group in call.arg_names:
|
||||
for name in group:
|
||||
if states.get(name) == TAINTED:
|
||||
names.append(name)
|
||||
return tuple(names)
|
||||
|
||||
|
||||
def _name_state(name: str, states: dict[str, str]) -> str:
|
||||
return states.get(name, UNKNOWN)
|
||||
|
||||
|
||||
def _source_in(text: str, facts: SemanticFacts) -> SourceFact | None:
|
||||
for source in facts.sources:
|
||||
for pattern in source.patterns:
|
||||
if pattern and pattern in text:
|
||||
return source
|
||||
return None
|
||||
|
||||
|
||||
def _sink_for(name: str, facts: SemanticFacts) -> SinkFact | None:
|
||||
for sink in facts.sinks:
|
||||
for call in sink.calls:
|
||||
if _call_matches(name, call):
|
||||
return sink
|
||||
return None
|
||||
|
||||
|
||||
def _sanitizer_for(call: CallSite, facts: SemanticFacts) -> str | None:
|
||||
for sanitizer in facts.sanitizers:
|
||||
if not any(_call_matches(call.name, name) for name in sanitizer.calls):
|
||||
continue
|
||||
if sanitizer.parameterized and call.extra_arg_is_sequence:
|
||||
return sanitizer.id
|
||||
if sanitizer.literal_format_arg is not None:
|
||||
index = sanitizer.literal_format_arg
|
||||
if index < len(call.arg_is_constant) and call.arg_is_constant[index] and _looks_like_format(call.arg_texts[index]):
|
||||
return sanitizer.id
|
||||
if not sanitizer.parameterized and sanitizer.literal_format_arg is None:
|
||||
return sanitizer.id
|
||||
return None
|
||||
|
||||
|
||||
def _looks_like_format(text: str) -> bool:
|
||||
stripped = text.strip()
|
||||
return len(stripped) >= 2 and stripped[0] in {"'", '"'} and stripped[-1] == stripped[0]
|
||||
|
||||
|
||||
def _known_callee(name: str, facts: SemanticFacts) -> bool:
|
||||
if _sink_for(name, facts) is not None:
|
||||
return True
|
||||
if _source_in(name, facts) is not None:
|
||||
return True
|
||||
return any(any(_call_matches(name, call) for call in sanitizer.calls) for sanitizer in facts.sanitizers)
|
||||
|
||||
|
||||
def _call_matches(name: str, fact_call: str) -> bool:
|
||||
if name == fact_call:
|
||||
return True
|
||||
if name.endswith("." + fact_call):
|
||||
return True
|
||||
if fact_call.endswith(name) and "." not in name:
|
||||
# "execute" matches "db.execute" via endswith .execute above; bare execute matches execute.
|
||||
return name == fact_call.split(".")[-1]
|
||||
return False
|
||||
|
||||
|
||||
def merge_joern_paths(existing: tuple[TaintPath, ...], extra: tuple[TaintPath, ...]) -> tuple[TaintPath, ...]:
|
||||
seen = {(item.path, item.sink_line, item.sink) for item in existing}
|
||||
merged = list(existing)
|
||||
for item in extra:
|
||||
key = (item.path, item.sink_line, item.sink)
|
||||
if key not in seen:
|
||||
merged.append(item)
|
||||
seen.add(key)
|
||||
return tuple(merged)
|
||||
Reference in New Issue
Block a user