feat(harness): add runtime and reachability mapping

This commit is contained in:
Marius Ciepluch
2026-06-18 07:57:06 +02:00
parent 33757db14a
commit f8c690999a
16 changed files with 1339 additions and 53 deletions
+33 -5
View File
@@ -6,8 +6,10 @@ from pathlib import Path
from .config import load_config
from .findings import quick_scan_findings, write_findings
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
from .index import build_code_chunks
from .preprocess import preprocess_repository
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
from .preprocess import preprocess_repository, write_preprocess_artifact
from .rank import rank_targets, write_rankings
from .reports import write_markdown_report
from .run import create_scan_run
@@ -43,12 +45,29 @@ def _scan(path: Path, config_path: Path, mode: str) -> int:
config = load_config(config_path if config_path.exists() else None)
run = create_scan_run(path, config)
_, targets = preprocess_repository(run.target, run.root / "preprocess" / "file_targets.json")
rankings = rank_targets(targets)
runtime = HarnessRuntime(
scan_id=run.scan_id,
config=config,
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl"),
)
write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json")
runtime.start(mode=mode, target=run.target)
static_hints = runtime.run_stage("static_mapping", lambda: _load_static_hints(config.static_analysis.sarif_paths))
write_static_hints(static_hints, run.root / "mapping" / "static_hints.json")
snapshot, targets = runtime.run_stage(
"preprocess",
lambda: preprocess_repository(run.target, run.root / "preprocess" / "file_targets.json", static_hints),
)
entry_points = runtime.run_stage("entry_point_mapping", lambda: analyze_entry_points(run.target, targets))
write_entry_points(entry_points, run.root / "mapping" / "entry_points.json")
targets = attach_reachability_hints(targets, entry_points)
write_preprocess_artifact(snapshot, targets, run.root / "preprocess" / "file_targets.json")
rankings = runtime.run_stage("rank", lambda: rank_targets(targets))
write_rankings(rankings, run.root / "rank" / "ranking.json")
findings = quick_scan_findings(run.target, targets, rankings)
findings = runtime.run_stage("quick_findings", lambda: quick_scan_findings(run.target, targets, rankings))
write_findings(findings, run.root / "findings.json")
write_markdown_report(findings, run.root / "report.md")
runtime.run_stage("report", lambda: write_markdown_report(findings, run.root / "report.md"))
runtime.finish(status="succeeded")
print(f"scan_id={run.scan_id}")
print(f"run_dir={run.root}")
print(f"file_targets={len(targets)}")
@@ -57,6 +76,15 @@ def _scan(path: Path, config_path: Path, mode: str) -> int:
return 0
def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]:
hints: list[object] = []
for sarif_path in sarif_paths:
path = Path(sarif_path)
if path.exists():
hints.extend(ingest_sarif(path))
return hints
def _index(path: Path, config_path: Path, chunk_lines: int) -> int:
if not path.exists() or not path.is_dir():
raise SystemExit(f"index path is not a directory: {path}")
+15
View File
@@ -46,6 +46,11 @@ class EvidenceConfig:
minimum_patch: str = "root_cause_explained"
@dataclass(frozen=True)
class StaticAnalysisConfig:
sarif_paths: tuple[str, ...] = ()
@dataclass(frozen=True)
class ResolvedConfig:
models: ModelConfig = ModelConfig()
@@ -53,6 +58,7 @@ class ResolvedConfig:
sandbox: SandboxConfig = SandboxConfig()
dynamic: DynamicConfig = DynamicConfig()
evidence: EvidenceConfig = EvidenceConfig()
static_analysis: StaticAnalysisConfig = StaticAnalysisConfig()
runs_dir: str = ".openultrasast/runs"
@@ -68,6 +74,7 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig:
sandbox=_load_sandbox(data.get("sandbox", {})),
dynamic=_load_dynamic(data.get("dynamic", {})),
evidence=_load_evidence(data.get("evidence", {})),
static_analysis=_load_static_analysis(data.get("static_analysis", {})),
runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"),
)
@@ -126,3 +133,11 @@ def _load_evidence(value: object) -> EvidenceConfig:
minimum_exploit=str(data.get("minimum_exploit", "crash_reproduced")),
minimum_patch=str(data.get("minimum_patch", "root_cause_explained")),
)
def _load_static_analysis(value: object) -> StaticAnalysisConfig:
data = _section(value)
paths = data.get("sarif_paths", [])
if not isinstance(paths, list):
paths = []
return StaticAnalysisConfig(sarif_paths=tuple(str(item) for item in paths))
+60 -5
View File
@@ -19,6 +19,10 @@ class StaticFinding:
evidence_level: str
rationale: str
line: int | None
function_name: str | None
reachability_status: str
reachability_evidence: list[dict[str, object]]
reachability_conditions: list[str]
tags: list[str]
ranking_priority: float
@@ -70,11 +74,9 @@ def quick_scan_findings(root: Path, targets: list[FileTarget], rankings: list[Ra
for target in targets:
text = _read_text(root / target.path)
for rule in PATTERN_RULES:
match = rule.pattern.search(text)
if match is None:
continue
ranking = ranking_by_path.get(target.path)
findings.append(_finding_from_match(target, rule, text, match.start(), ranking))
for match in rule.pattern.finditer(text):
findings.append(_finding_from_match(target, rule, text, match.start(), ranking))
return sorted(findings, key=lambda item: (_severity_sort(item.severity), -item.ranking_priority, item.path))
@@ -103,6 +105,8 @@ def _finding_from_match(
ranking: RankingScore | None,
) -> StaticFinding:
line = text.count("\n", 0, offset) + 1
reachability_status, function_name, reachability_evidence = _reachability_for_line(target, line)
priority = ranking.priority if ranking is not None else 1.0
return StaticFinding(
finding_id=f"{rule.rule_id}:{target.path}:{line}",
path=target.path,
@@ -112,11 +116,62 @@ def _finding_from_match(
evidence_level="static_corroboration",
rationale=f"Static pattern {rule.rule_id} matched line {line}; manual verification still required.",
line=line,
function_name=function_name,
reachability_status=reachability_status,
reachability_evidence=reachability_evidence,
reachability_conditions=_reachability_conditions(reachability_evidence),
tags=sorted(set(target.tags) | set(rule.tags)),
ranking_priority=ranking.priority if ranking is not None else 1.0,
ranking_priority=_finding_priority(priority, reachability_status),
)
def _reachability_for_line(target: FileTarget, line: int) -> tuple[str, str | None, list[dict[str, object]]]:
hints = target.reachability_hints
concrete = [hint for hint in hints if isinstance(hint.get("line"), int)]
matching = [hint for hint in concrete if _line_in_hint(line, hint)]
if matching:
return "reachable", _function_name(matching), matching
if concrete:
return "unknown", None, []
inferred = [hint for hint in hints if hint.get("line") is None]
if inferred:
return "inferred-file-surface", None, inferred
return "unknown", None, []
def _line_in_hint(line: int, hint: dict[str, object]) -> bool:
start = hint.get("line")
end = hint.get("end_line")
if not isinstance(start, int):
return False
if not isinstance(end, int):
end = start
return start <= line <= end
def _function_name(hints: list[dict[str, object]]) -> str | None:
for hint in hints:
function_name = hint.get("function_name")
if isinstance(function_name, str) and function_name:
return function_name
return None
def _reachability_conditions(hints: list[dict[str, object]]) -> list[str]:
conditions: list[str] = []
for hint in hints:
value = hint.get("conditions")
if isinstance(value, list):
conditions.extend(str(item) for item in value if item)
return sorted(set(conditions))
def _finding_priority(priority: float, reachability_status: str) -> float:
if reachability_status == "unknown":
return min(priority, 1.5)
return priority
def _read_text(path: Path) -> str:
try:
return path.read_text(errors="ignore")
+187
View File
@@ -0,0 +1,187 @@
from __future__ import annotations
import hashlib
import json
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable, Literal, Protocol
from .config import ResolvedConfig
EventType = Literal[
"task_start",
"stage_start",
"before_model",
"after_model",
"before_tool",
"after_tool",
"stage_end",
"task_end",
]
ContractMode = Literal["strict", "warn"]
class HarnessContractError(ValueError):
"""Raised when a processor violates its declared state contract."""
@dataclass(frozen=True)
class HarnessEvent:
event_type: EventType
scan_id: str
stage: str | None
timestamp: str
payload: dict[str, Any] = field(default_factory=dict)
@dataclass(frozen=True)
class ProcessorSpec:
name: str
version: str
handles: tuple[EventType, ...]
reads: tuple[str, ...] = ()
writes: tuple[str, ...] = ()
class Processor(Protocol):
spec: ProcessorSpec
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@dataclass(frozen=True)
class SerializedHarnessConfig:
contract_mode: ContractMode
model_roles: dict[str, str | None]
processor_versions: dict[str, str]
prompt_hashes: dict[str, str]
class HarnessTraceWriter:
def __init__(self, path: Path) -> None:
self.path = path
self.path.parent.mkdir(parents=True, exist_ok=True)
def append(self, event: HarnessEvent) -> None:
with self.path.open("a") as handle:
handle.write(json.dumps(asdict(event), sort_keys=True) + "\n")
class HarnessRuntime:
def __init__(
self,
*,
scan_id: str,
config: ResolvedConfig,
trace_writer: HarnessTraceWriter,
processors: list[Processor] | None = None,
contract_mode: ContractMode = "strict",
) -> None:
self.scan_id = scan_id
self.config = config
self.trace_writer = trace_writer
self.processors = processors or []
self.contract_mode = contract_mode
self.state: dict[str, Any] = {"degradations": []}
def start(self, *, mode: str, target: Path) -> None:
self.emit("task_start", None, {"mode": mode, "target": str(target)})
def finish(self, *, status: str) -> None:
self.emit("task_end", None, {"status": status})
def run_stage(self, name: str, callback: Callable[[], Any]) -> Any:
self.emit("stage_start", name, {})
try:
result = callback()
except Exception as exc:
self.emit("stage_end", name, {"status": "failed", "error": f"{type(exc).__name__}: {exc}"})
raise
self.emit("stage_end", name, {"status": "succeeded"})
return result
def emit(self, event_type: EventType, stage: str | None, payload: dict[str, Any]) -> None:
event = HarnessEvent(
event_type=event_type,
scan_id=self.scan_id,
stage=stage,
timestamp=datetime.now(UTC).isoformat(),
payload=payload,
)
self.trace_writer.append(event)
for processor in self.processors:
if event_type in processor.spec.handles:
self._run_processor(processor, event)
def _run_processor(self, processor: Processor, event: HarnessEvent) -> None:
before = set(self.state)
missing_reads = [key for key in processor.spec.reads if key not in self.state]
if missing_reads:
self._contract_violation(processor.spec.name, f"missing reads: {', '.join(missing_reads)}")
return
updates = processor.handle(event, dict(self.state)) or {}
if not isinstance(updates, dict):
self._contract_violation(processor.spec.name, "processor returned a non-dict update")
return
undeclared = sorted(set(updates) - set(processor.spec.writes))
if undeclared:
self._contract_violation(processor.spec.name, f"undeclared writes: {', '.join(undeclared)}")
return
self.state.update(updates)
changed = sorted(set(self.state) - before | set(updates))
self.trace_writer.append(
HarnessEvent(
event_type=event.event_type,
scan_id=self.scan_id,
stage=event.stage,
timestamp=datetime.now(UTC).isoformat(),
payload={"processor": processor.spec.name, "changed_state": changed},
)
)
def _contract_violation(self, processor_name: str, reason: str) -> None:
message = f"processor {processor_name} contract violation: {reason}"
if self.contract_mode == "strict":
raise HarnessContractError(message)
self.state.setdefault("degradations", []).append(message)
self.trace_writer.append(
HarnessEvent(
event_type="stage_end",
scan_id=self.scan_id,
stage="processor_contract",
timestamp=datetime.now(UTC).isoformat(),
payload={"status": "degraded", "reason": message},
)
)
def write_harness_config(
*,
config: ResolvedConfig,
processors: list[Processor],
contract_mode: ContractMode,
path: Path,
prompts: dict[str, str] | None = None,
) -> None:
prompts = prompts or {}
serialized = SerializedHarnessConfig(
contract_mode=contract_mode,
model_roles={
"ranker": config.models.ranker,
"hunter": config.models.hunter,
"verifier": config.models.verifier,
"patcher": config.models.patcher,
},
processor_versions={processor.spec.name: processor.spec.version for processor in processors},
prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())},
)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(asdict(serialized), indent=2, sort_keys=True) + "\n")
def _hash_text(text: str) -> str:
return hashlib.sha256(text.encode()).hexdigest()
+455
View File
@@ -0,0 +1,455 @@
from __future__ import annotations
import ast
import json
from dataclasses import asdict, dataclass, replace
from pathlib import Path
from typing import Any, Literal
from .preprocess import FileTarget
MappingKind = Literal["semgrep", "codeql", "differential", "sharp_edge"]
AccessLevel = Literal[
"public",
"authenticated",
"role-restricted",
"contract-only/callback",
"local-only",
"review-required",
]
@dataclass(frozen=True)
class StaticHint:
analyzer: str
rule_id: str
title: str
severity: str
path: str
start_line: int | None
message: str
fingerprint: str | None
provenance: str
evidence_candidate: bool = True
@dataclass(frozen=True)
class MappingTaskRecord:
kind: MappingKind
path: str
rule_id: str
task: str
rationale: str
@dataclass(frozen=True)
class DifferentialMappingRecord:
path: str
change_kind: str
trust_boundary: str | None
rationale: str
@dataclass(frozen=True)
class SharpEdgeRecord:
path: str
category: str
surface: str
rationale: str
@dataclass(frozen=True)
class EntryPointRecord:
path: str
line: int | None
end_line: int | None
function_name: str | None
name: str
kind: str
access_level: AccessLevel
trust_boundary: str
access_evidence: list[str]
conditions: list[str]
provenance: str
rationale: str
def ingest_sarif(path: Path) -> list[StaticHint]:
payload = json.loads(path.read_text())
hints: list[StaticHint] = []
for run in _items(payload.get("runs")):
tool = run.get("tool") if isinstance(run, dict) else {}
driver = tool.get("driver") if isinstance(tool, dict) else {}
analyzer = _normalise_analyzer(str(driver.get("name", "sarif"))) if isinstance(driver, dict) else "sarif"
rules = _rules_by_id(driver.get("rules", []) if isinstance(driver, dict) else [])
for result in _items(run.get("results") if isinstance(run, dict) else []):
hint = _hint_from_result(result, analyzer, rules)
if hint is not None:
hints.append(hint)
return sorted(hints, key=lambda item: (item.path, item.start_line or 0, item.rule_id))
def attach_static_hints(targets: list[FileTarget], hints: list[StaticHint]) -> list[FileTarget]:
hints_by_path: dict[str, list[dict[str, object]]] = {}
for hint in hints:
hints_by_path.setdefault(hint.path, []).append(asdict(hint))
return [replace(target, static_hints=hints_by_path.get(target.path, target.static_hints)) for target in targets]
def analyze_entry_points(root: Path, targets: list[FileTarget]) -> list[EntryPointRecord]:
records: list[EntryPointRecord] = []
for target in targets:
path = root / target.path
text = path.read_text(errors="ignore")
records.extend(_language_entry_points(target, text))
records.extend(_tag_entry_points(target))
return sorted(records, key=lambda item: (item.path, item.line is None, item.line or 0, item.name))
def attach_reachability_hints(targets: list[FileTarget], entry_points: list[EntryPointRecord]) -> list[FileTarget]:
hints_by_path: dict[str, list[dict[str, object]]] = {}
for entry_point in entry_points:
hints_by_path.setdefault(entry_point.path, []).append(asdict(entry_point))
return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets]
def verifier_evidence_candidates(hints: list[StaticHint], *, path: str | None = None) -> list[dict[str, object]]:
selected = [hint for hint in hints if hint.evidence_candidate and (path is None or hint.path == path)]
return [
{
"source": hint.analyzer,
"rule_id": hint.rule_id,
"path": hint.path,
"line": hint.start_line,
"severity": hint.severity,
"message": hint.message,
"provenance": hint.provenance,
}
for hint in selected
]
def semgrep_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
return [
MappingTaskRecord(
kind="semgrep",
path=hint.path,
rule_id=hint.rule_id,
task="pattern_variant_review",
rationale="Use Semgrep evidence to check variants and rule precision before verifier promotion.",
)
for hint in hints
if hint.analyzer == "semgrep"
]
def codeql_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
return [
MappingTaskRecord(
kind="codeql",
path=hint.path,
rule_id=hint.rule_id,
task="source_sink_sanitizer_path_review",
rationale="Use CodeQL evidence to inspect source, sink, sanitizer, and path explanation.",
)
for hint in hints
if hint.analyzer == "codeql"
]
def differential_mapping_record(path: str, *, change_kind: str, trust_boundary: str | None, rationale: str) -> DifferentialMappingRecord:
return DifferentialMappingRecord(path=path, change_kind=change_kind, trust_boundary=trust_boundary, rationale=rationale)
def sharp_edge_record(path: str, *, category: str, surface: str, rationale: str) -> SharpEdgeRecord:
return SharpEdgeRecord(path=path, category=category, surface=surface, rationale=rationale)
def write_static_hints(hints: list[StaticHint], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n")
def write_entry_points(entry_points: list[EntryPointRecord], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"entry_points": [asdict(entry_point) for entry_point in entry_points]}, indent=2, sort_keys=True) + "\n")
def _language_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
if target.language == "python":
return _python_entry_points(target, text)
if target.language in {"javascript", "typescript"}:
return _js_entry_points(target, text)
if target.language in {"c", "cpp"}:
return _c_entry_points(target, text)
if target.language == "solidity":
return _solidity_entry_points(target, text)
return []
def _python_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
records: list[EntryPointRecord] = []
try:
tree = ast.parse(text)
except SyntaxError:
tree = None
if tree is not None:
for node in ast.walk(tree):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
decorators = [_safe_unparse(decorator) for decorator in node.decorator_list]
route = next((decorator for decorator in decorators if _looks_like_route_decorator(decorator)), None)
if route is not None:
access, evidence = _python_route_access(decorators)
conditions = _python_conditions(node, decorators)
records.append(
_entry(
target,
node.lineno,
getattr(node, "end_lineno", node.lineno),
node.name,
node.name,
"route",
access,
"http_request",
evidence or [route],
conditions,
)
)
if any("click.command" in decorator or ".command" in decorator for decorator in decorators):
records.append(
_entry(
target,
node.lineno,
getattr(node, "end_lineno", node.lineno),
node.name,
node.name,
"cli",
"local-only",
"process_invocation",
decorators,
_python_conditions(node, decorators),
)
)
for number, line in enumerate(text.splitlines(), start=1):
stripped = line.strip()
if stripped.startswith("if __name__") and "__main__" in stripped:
records.append(_entry(target, number, number, "__main__", "__main__", "cli", "local-only", "process_invocation", [stripped], []))
if "argparse." in stripped or "click.command" in stripped:
records.append(_entry(target, number, number, None, "cli_parser", "cli", "local-only", "process_invocation", [stripped], []))
return records
def _js_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
records: list[EntryPointRecord] = []
route_tokens = (".get(", ".post(", ".put(", ".delete(", ".patch(", ".use(")
for number, line in enumerate(text.splitlines(), start=1):
stripped = line.strip()
if any(token in stripped for token in route_tokens) and any(prefix in stripped for prefix in ("app", "router", "server")):
records.append(_entry(target, number, number, _js_handler_name(stripped), "http_handler", "route", "public", "http_request", [stripped], _line_conditions(stripped)))
if "process.argv" in stripped:
records.append(_entry(target, number, number, None, "process.argv", "cli", "local-only", "process_invocation", [stripped], _line_conditions(stripped)))
return records
def _c_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
records: list[EntryPointRecord] = []
for number, line in enumerate(text.splitlines(), start=1):
stripped = line.strip()
if " main(" in f" {stripped}" or stripped.startswith("main("):
records.append(_entry(target, number, number, "main", "main", "cli", "local-only", "process_invocation", [stripped], _line_conditions(stripped)))
if "LLVMFuzzerTestOneInput" in stripped:
records.append(_entry(target, number, number, "LLVMFuzzerTestOneInput", "LLVMFuzzerTestOneInput", "fuzz", "public", "fuzzer_input", [stripped], _line_conditions(stripped)))
return records
def _solidity_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
records: list[EntryPointRecord] = []
for number, line in enumerate(text.splitlines(), start=1):
stripped = line.strip()
if not stripped.startswith("function ") or not any(token in stripped for token in (" public", " external")):
continue
if " view" in stripped or " pure" in stripped:
continue
evidence = [token for token in ("onlyOwner", "onlyRole", "requiresAuth") if token in stripped]
conditions = [token for token in ("whenNotPaused", "whenPaused", "featureEnabled") if token in stripped]
access = "role-restricted" if evidence else "public"
name = stripped.removeprefix("function ").split("(", 1)[0]
if "callback" in name.lower() or name.startswith("onERC"):
access = "contract-only/callback"
evidence.append("callback naming convention")
records.append(_entry(target, number, number, name, name, "smart_contract_state_change", access, "contract_call", evidence or ["external/public function"], conditions))
return records
def _tag_entry_points(target: FileTarget) -> list[EntryPointRecord]:
records: list[EntryPointRecord] = []
tags = set(target.tags)
if "parser" in tags or "deserialization" in tags:
records.append(_entry(target, None, None, None, "parser_input", "parser", "public", "attacker_controlled_input", ["parser/deserialization file tag"], []))
if "network_entry" in tags:
records.append(_entry(target, None, None, None, "network_surface", "route", "public", "network_request", ["network entry file tag"], []))
if "auth_boundary" in tags:
records.append(_entry(target, None, None, None, "auth_boundary", "privileged_surface", "authenticated", "identity_boundary", ["auth boundary file tag"], []))
if "syscall_entry" in tags or "filesystem_entry" in tags:
records.append(_entry(target, None, None, None, "local_privileged_boundary", "privileged_surface", "local-only", "local_process", ["syscall/filesystem file tag"], []))
return records
def _entry(
target: FileTarget,
line: int | None,
end_line: int | None,
function_name: str | None,
name: str,
kind: str,
access_level: AccessLevel,
trust_boundary: str,
access_evidence: list[str],
conditions: list[str],
) -> EntryPointRecord:
return EntryPointRecord(
path=target.path,
line=line,
end_line=end_line,
function_name=function_name,
name=name,
kind=kind,
access_level=access_level,
trust_boundary=trust_boundary,
access_evidence=access_evidence,
conditions=conditions,
provenance=f"entrypoint:{kind}:{access_level}",
rationale=f"{kind} surface classified as {access_level} at {trust_boundary}",
)
def _looks_like_route_decorator(line: str) -> bool:
return any(token in line for token in ("route", ".get", ".post", ".put", ".delete", ".patch"))
def _safe_unparse(node: ast.AST) -> str:
try:
return ast.unparse(node)
except (AttributeError, ValueError):
return ""
def _python_route_access(decorators: list[str]) -> tuple[AccessLevel, list[str]]:
role_tokens = ("role", "permission", "admin", "owner")
auth_tokens = ("login_required", "authenticated", "auth", "jwt_required")
access_decorators = [decorator for decorator in decorators if not _looks_like_route_decorator(decorator)]
evidence = [decorator for decorator in access_decorators if any(token in decorator.lower() for token in role_tokens + auth_tokens)]
lowered = "\n".join(evidence).lower()
if any(token in lowered for token in role_tokens):
return "role-restricted", evidence
if any(token in lowered for token in auth_tokens):
return "authenticated", evidence
return "public", evidence
def _python_conditions(node: ast.FunctionDef | ast.AsyncFunctionDef, decorators: list[str]) -> list[str]:
conditions = [decorator for decorator in decorators if _looks_conditional(decorator)]
for child in ast.walk(node):
if isinstance(child, ast.If):
test = _safe_unparse(child.test)
if _looks_conditional(test):
conditions.append(test)
return sorted(set(conditions))
def _line_conditions(line: str) -> list[str]:
return [line] if _looks_conditional(line) else []
def _looks_conditional(text: str) -> bool:
lowered = text.lower()
return any(token in lowered for token in ("feature", "flag", "toggle", "enabled", "disabled", "experiment", "beta", "rollout", "paused"))
def _js_handler_name(line: str) -> str | None:
if "function " in line:
return line.split("function ", 1)[1].split("(", 1)[0].strip() or None
if "=>" in line:
return "arrow_handler"
return None
def _hint_from_result(result: object, analyzer: str, rules: dict[str, dict[str, object]]) -> StaticHint | None:
if not isinstance(result, dict):
return None
rule_id = str(result.get("ruleId", "unknown"))
location = _primary_location(result)
if location is None:
return None
artifact = location.get("physicalLocation", {}).get("artifactLocation", {})
region = location.get("physicalLocation", {}).get("region", {})
uri = artifact.get("uri") if isinstance(artifact, dict) else None
if not isinstance(uri, str) or not uri:
return None
rule = rules.get(rule_id, {})
message = result.get("message", {})
message_text = message.get("text") if isinstance(message, dict) else None
return StaticHint(
analyzer=analyzer,
rule_id=rule_id,
title=str(rule.get("name") or rule.get("shortDescription") or rule_id),
severity=_severity(result, rule),
path=uri.lstrip("./"),
start_line=region.get("startLine") if isinstance(region.get("startLine"), int) else None,
message=message_text if isinstance(message_text, str) else rule_id,
fingerprint=_fingerprint(result),
provenance=f"sarif:{analyzer}:{rule_id}",
)
def _primary_location(result: dict[str, object]) -> dict[str, Any] | None:
locations = result.get("locations")
if not isinstance(locations, list) or not locations or not isinstance(locations[0], dict):
return None
return locations[0]
def _rules_by_id(rules: object) -> dict[str, dict[str, object]]:
by_id: dict[str, dict[str, object]] = {}
for rule in _items(rules):
rule_id = rule.get("id") if isinstance(rule, dict) else None
if isinstance(rule_id, str):
by_id[rule_id] = rule
return by_id
def _severity(result: dict[str, object], rule: dict[str, object]) -> str:
level = result.get("level")
if level == "error":
return "high"
if level == "warning":
return "medium"
properties = rule.get("properties")
if isinstance(properties, dict) and "security-severity" in properties:
try:
return "high" if float(properties["security-severity"]) >= 7.0 else "medium"
except (TypeError, ValueError):
return "medium"
return "low"
def _fingerprint(result: dict[str, object]) -> str | None:
for key in ("partialFingerprints", "fingerprints"):
value = result.get(key)
if isinstance(value, dict) and value:
first = next(iter(value.values()))
return str(first)
return None
def _normalise_analyzer(name: str) -> str:
lowered = name.lower()
if "semgrep" in lowered:
return "semgrep"
if "codeql" in lowered:
return "codeql"
return lowered.replace(" ", "_") or "sarif"
def _items(value: object) -> list[object]:
return value if isinstance(value, list) else []
+22 -5
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import fnmatch
import json
import subprocess
from dataclasses import asdict, dataclass
from dataclasses import asdict, dataclass, field
from pathlib import Path
@@ -29,6 +29,7 @@ LANGUAGE_BY_EXTENSION = {
".php": "php",
".cs": "csharp",
".swift": "swift",
".sol": "solidity",
}
MEMORY_UNSAFE_LANGUAGES = {"c", "cpp"}
@@ -50,12 +51,22 @@ class FileTarget:
loc: int
tags: list[str]
has_fuzz_entry_point: bool
static_hints: list[dict[str, object]] = field(default_factory=list)
reachability_hints: list[dict[str, object]] = field(default_factory=list)
def preprocess_repository(root: Path, output_path: Path | None = None) -> tuple[RepoSnapshot, list[FileTarget]]:
def preprocess_repository(
root: Path,
output_path: Path | None = None,
static_hints: list[object] | None = None,
) -> tuple[RepoSnapshot, list[FileTarget]]:
resolved = root.resolve()
files = enumerate_source_files(resolved)
targets = [build_file_target(resolved, path) for path in files]
if static_hints:
from .mapping import attach_static_hints
targets = attach_static_hints(targets, static_hints) # type: ignore[arg-type]
languages: dict[str, int] = {}
for target in targets:
languages[target.language] = languages.get(target.language, 0) + 1
@@ -68,13 +79,17 @@ def preprocess_repository(root: Path, output_path: Path | None = None) -> tuple[
)
if output_path is not None:
output_path.parent.mkdir(parents=True, exist_ok=True)
payload = {"snapshot": asdict(snapshot), "file_targets": [asdict(target) for target in targets]}
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
write_preprocess_artifact(snapshot, targets, output_path)
return snapshot, targets
def write_preprocess_artifact(snapshot: RepoSnapshot, targets: list[FileTarget], output_path: Path) -> None:
output_path.parent.mkdir(parents=True, exist_ok=True)
payload = {"snapshot": asdict(snapshot), "file_targets": [asdict(target) for target in targets]}
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
def enumerate_source_files(root: Path) -> list[Path]:
patterns = _load_ignore_patterns(root)
paths: list[Path] = []
@@ -97,6 +112,8 @@ def build_file_target(root: Path, path: Path) -> FileTarget:
loc=count_loc(text),
tags=tags,
has_fuzz_entry_point="LLVMFuzzerTestOneInput" in text,
static_hints=[],
reachability_hints=[],
)
+23
View File
@@ -105,6 +105,10 @@ def heuristic_rank(target: FileTarget) -> RankingScore:
if target.loc >= 500:
influence = max(influence, 3)
boosts.append("large_file")
reachability_boost = _reachability_from_hints(target.reachability_hints)
if reachability_boost:
reachability = max(reachability, reachability_boost)
boosts.append("entry_point_reachable")
return RankingScore(
path=target.path,
@@ -212,4 +216,23 @@ def _target_prompt_payload(target: FileTarget) -> dict[str, object]:
"loc": target.loc,
"tags": target.tags,
"has_fuzz_entry_point": target.has_fuzz_entry_point,
"reachability_hints": target.reachability_hints,
}
def _reachability_from_hints(hints: list[dict[str, object]]) -> int:
score = 0
for hint in hints:
access_level = hint.get("access_level")
kind = hint.get("kind")
if access_level == "public":
score = max(score, 5 if kind in {"route", "parser", "fuzz"} else 4)
elif access_level in {"authenticated", "contract-only/callback"}:
score = max(score, 4)
elif access_level == "role-restricted":
score = max(score, 3)
elif access_level == "review-required":
score = max(score, 2)
elif access_level == "local-only":
score = max(score, 2)
return score
+3
View File
@@ -18,9 +18,12 @@ def write_markdown_report(findings: list[StaticFinding], path: Path) -> None:
f"- ID: `{finding.finding_id}`",
f"- Path: `{finding.path}`",
f"- Line: `{finding.line}`",
f"- Function: `{finding.function_name or 'unknown'}`",
f"- Severity: `{finding.severity}`",
f"- Confidence: `{finding.confidence}`",
f"- Evidence level: `{finding.evidence_level}`",
f"- Reachability: `{finding.reachability_status}`",
f"- Conditions: `{', '.join(finding.reachability_conditions) or 'none'}`",
f"- Ranking priority: `{finding.ranking_priority}`",
f"- Tags: `{', '.join(finding.tags)}`",
"",