mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
feat(harness): add runtime and reachability mapping
This commit is contained in:
@@ -6,8 +6,10 @@ from pathlib import Path
|
||||
|
||||
from .config import load_config
|
||||
from .findings import quick_scan_findings, write_findings
|
||||
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||
from .index import build_code_chunks
|
||||
from .preprocess import preprocess_repository
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
||||
from .preprocess import preprocess_repository, write_preprocess_artifact
|
||||
from .rank import rank_targets, write_rankings
|
||||
from .reports import write_markdown_report
|
||||
from .run import create_scan_run
|
||||
@@ -43,12 +45,29 @@ def _scan(path: Path, config_path: Path, mode: str) -> int:
|
||||
|
||||
config = load_config(config_path if config_path.exists() else None)
|
||||
run = create_scan_run(path, config)
|
||||
_, targets = preprocess_repository(run.target, run.root / "preprocess" / "file_targets.json")
|
||||
rankings = rank_targets(targets)
|
||||
runtime = HarnessRuntime(
|
||||
scan_id=run.scan_id,
|
||||
config=config,
|
||||
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl"),
|
||||
)
|
||||
write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json")
|
||||
runtime.start(mode=mode, target=run.target)
|
||||
static_hints = runtime.run_stage("static_mapping", lambda: _load_static_hints(config.static_analysis.sarif_paths))
|
||||
write_static_hints(static_hints, run.root / "mapping" / "static_hints.json")
|
||||
snapshot, targets = runtime.run_stage(
|
||||
"preprocess",
|
||||
lambda: preprocess_repository(run.target, run.root / "preprocess" / "file_targets.json", static_hints),
|
||||
)
|
||||
entry_points = runtime.run_stage("entry_point_mapping", lambda: analyze_entry_points(run.target, targets))
|
||||
write_entry_points(entry_points, run.root / "mapping" / "entry_points.json")
|
||||
targets = attach_reachability_hints(targets, entry_points)
|
||||
write_preprocess_artifact(snapshot, targets, run.root / "preprocess" / "file_targets.json")
|
||||
rankings = runtime.run_stage("rank", lambda: rank_targets(targets))
|
||||
write_rankings(rankings, run.root / "rank" / "ranking.json")
|
||||
findings = quick_scan_findings(run.target, targets, rankings)
|
||||
findings = runtime.run_stage("quick_findings", lambda: quick_scan_findings(run.target, targets, rankings))
|
||||
write_findings(findings, run.root / "findings.json")
|
||||
write_markdown_report(findings, run.root / "report.md")
|
||||
runtime.run_stage("report", lambda: write_markdown_report(findings, run.root / "report.md"))
|
||||
runtime.finish(status="succeeded")
|
||||
print(f"scan_id={run.scan_id}")
|
||||
print(f"run_dir={run.root}")
|
||||
print(f"file_targets={len(targets)}")
|
||||
@@ -57,6 +76,15 @@ def _scan(path: Path, config_path: Path, mode: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]:
|
||||
hints: list[object] = []
|
||||
for sarif_path in sarif_paths:
|
||||
path = Path(sarif_path)
|
||||
if path.exists():
|
||||
hints.extend(ingest_sarif(path))
|
||||
return hints
|
||||
|
||||
|
||||
def _index(path: Path, config_path: Path, chunk_lines: int) -> int:
|
||||
if not path.exists() or not path.is_dir():
|
||||
raise SystemExit(f"index path is not a directory: {path}")
|
||||
|
||||
@@ -46,6 +46,11 @@ class EvidenceConfig:
|
||||
minimum_patch: str = "root_cause_explained"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StaticAnalysisConfig:
|
||||
sarif_paths: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedConfig:
|
||||
models: ModelConfig = ModelConfig()
|
||||
@@ -53,6 +58,7 @@ class ResolvedConfig:
|
||||
sandbox: SandboxConfig = SandboxConfig()
|
||||
dynamic: DynamicConfig = DynamicConfig()
|
||||
evidence: EvidenceConfig = EvidenceConfig()
|
||||
static_analysis: StaticAnalysisConfig = StaticAnalysisConfig()
|
||||
runs_dir: str = ".openultrasast/runs"
|
||||
|
||||
|
||||
@@ -68,6 +74,7 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig:
|
||||
sandbox=_load_sandbox(data.get("sandbox", {})),
|
||||
dynamic=_load_dynamic(data.get("dynamic", {})),
|
||||
evidence=_load_evidence(data.get("evidence", {})),
|
||||
static_analysis=_load_static_analysis(data.get("static_analysis", {})),
|
||||
runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"),
|
||||
)
|
||||
|
||||
@@ -126,3 +133,11 @@ def _load_evidence(value: object) -> EvidenceConfig:
|
||||
minimum_exploit=str(data.get("minimum_exploit", "crash_reproduced")),
|
||||
minimum_patch=str(data.get("minimum_patch", "root_cause_explained")),
|
||||
)
|
||||
|
||||
|
||||
def _load_static_analysis(value: object) -> StaticAnalysisConfig:
|
||||
data = _section(value)
|
||||
paths = data.get("sarif_paths", [])
|
||||
if not isinstance(paths, list):
|
||||
paths = []
|
||||
return StaticAnalysisConfig(sarif_paths=tuple(str(item) for item in paths))
|
||||
|
||||
@@ -19,6 +19,10 @@ class StaticFinding:
|
||||
evidence_level: str
|
||||
rationale: str
|
||||
line: int | None
|
||||
function_name: str | None
|
||||
reachability_status: str
|
||||
reachability_evidence: list[dict[str, object]]
|
||||
reachability_conditions: list[str]
|
||||
tags: list[str]
|
||||
ranking_priority: float
|
||||
|
||||
@@ -70,11 +74,9 @@ def quick_scan_findings(root: Path, targets: list[FileTarget], rankings: list[Ra
|
||||
for target in targets:
|
||||
text = _read_text(root / target.path)
|
||||
for rule in PATTERN_RULES:
|
||||
match = rule.pattern.search(text)
|
||||
if match is None:
|
||||
continue
|
||||
ranking = ranking_by_path.get(target.path)
|
||||
findings.append(_finding_from_match(target, rule, text, match.start(), ranking))
|
||||
for match in rule.pattern.finditer(text):
|
||||
findings.append(_finding_from_match(target, rule, text, match.start(), ranking))
|
||||
return sorted(findings, key=lambda item: (_severity_sort(item.severity), -item.ranking_priority, item.path))
|
||||
|
||||
|
||||
@@ -103,6 +105,8 @@ def _finding_from_match(
|
||||
ranking: RankingScore | None,
|
||||
) -> StaticFinding:
|
||||
line = text.count("\n", 0, offset) + 1
|
||||
reachability_status, function_name, reachability_evidence = _reachability_for_line(target, line)
|
||||
priority = ranking.priority if ranking is not None else 1.0
|
||||
return StaticFinding(
|
||||
finding_id=f"{rule.rule_id}:{target.path}:{line}",
|
||||
path=target.path,
|
||||
@@ -112,11 +116,62 @@ def _finding_from_match(
|
||||
evidence_level="static_corroboration",
|
||||
rationale=f"Static pattern {rule.rule_id} matched line {line}; manual verification still required.",
|
||||
line=line,
|
||||
function_name=function_name,
|
||||
reachability_status=reachability_status,
|
||||
reachability_evidence=reachability_evidence,
|
||||
reachability_conditions=_reachability_conditions(reachability_evidence),
|
||||
tags=sorted(set(target.tags) | set(rule.tags)),
|
||||
ranking_priority=ranking.priority if ranking is not None else 1.0,
|
||||
ranking_priority=_finding_priority(priority, reachability_status),
|
||||
)
|
||||
|
||||
|
||||
def _reachability_for_line(target: FileTarget, line: int) -> tuple[str, str | None, list[dict[str, object]]]:
|
||||
hints = target.reachability_hints
|
||||
concrete = [hint for hint in hints if isinstance(hint.get("line"), int)]
|
||||
matching = [hint for hint in concrete if _line_in_hint(line, hint)]
|
||||
if matching:
|
||||
return "reachable", _function_name(matching), matching
|
||||
if concrete:
|
||||
return "unknown", None, []
|
||||
inferred = [hint for hint in hints if hint.get("line") is None]
|
||||
if inferred:
|
||||
return "inferred-file-surface", None, inferred
|
||||
return "unknown", None, []
|
||||
|
||||
|
||||
def _line_in_hint(line: int, hint: dict[str, object]) -> bool:
|
||||
start = hint.get("line")
|
||||
end = hint.get("end_line")
|
||||
if not isinstance(start, int):
|
||||
return False
|
||||
if not isinstance(end, int):
|
||||
end = start
|
||||
return start <= line <= end
|
||||
|
||||
|
||||
def _function_name(hints: list[dict[str, object]]) -> str | None:
|
||||
for hint in hints:
|
||||
function_name = hint.get("function_name")
|
||||
if isinstance(function_name, str) and function_name:
|
||||
return function_name
|
||||
return None
|
||||
|
||||
|
||||
def _reachability_conditions(hints: list[dict[str, object]]) -> list[str]:
|
||||
conditions: list[str] = []
|
||||
for hint in hints:
|
||||
value = hint.get("conditions")
|
||||
if isinstance(value, list):
|
||||
conditions.extend(str(item) for item in value if item)
|
||||
return sorted(set(conditions))
|
||||
|
||||
|
||||
def _finding_priority(priority: float, reachability_status: str) -> float:
|
||||
if reachability_status == "unknown":
|
||||
return min(priority, 1.5)
|
||||
return priority
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text(errors="ignore")
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Literal, Protocol
|
||||
|
||||
from .config import ResolvedConfig
|
||||
|
||||
EventType = Literal[
|
||||
"task_start",
|
||||
"stage_start",
|
||||
"before_model",
|
||||
"after_model",
|
||||
"before_tool",
|
||||
"after_tool",
|
||||
"stage_end",
|
||||
"task_end",
|
||||
]
|
||||
|
||||
ContractMode = Literal["strict", "warn"]
|
||||
|
||||
|
||||
class HarnessContractError(ValueError):
|
||||
"""Raised when a processor violates its declared state contract."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessEvent:
|
||||
event_type: EventType
|
||||
scan_id: str
|
||||
stage: str | None
|
||||
timestamp: str
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessorSpec:
|
||||
name: str
|
||||
version: str
|
||||
handles: tuple[EventType, ...]
|
||||
reads: tuple[str, ...] = ()
|
||||
writes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class Processor(Protocol):
|
||||
spec: ProcessorSpec
|
||||
|
||||
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SerializedHarnessConfig:
|
||||
contract_mode: ContractMode
|
||||
model_roles: dict[str, str | None]
|
||||
processor_versions: dict[str, str]
|
||||
prompt_hashes: dict[str, str]
|
||||
|
||||
|
||||
class HarnessTraceWriter:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def append(self, event: HarnessEvent) -> None:
|
||||
with self.path.open("a") as handle:
|
||||
handle.write(json.dumps(asdict(event), sort_keys=True) + "\n")
|
||||
|
||||
|
||||
class HarnessRuntime:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
scan_id: str,
|
||||
config: ResolvedConfig,
|
||||
trace_writer: HarnessTraceWriter,
|
||||
processors: list[Processor] | None = None,
|
||||
contract_mode: ContractMode = "strict",
|
||||
) -> None:
|
||||
self.scan_id = scan_id
|
||||
self.config = config
|
||||
self.trace_writer = trace_writer
|
||||
self.processors = processors or []
|
||||
self.contract_mode = contract_mode
|
||||
self.state: dict[str, Any] = {"degradations": []}
|
||||
|
||||
def start(self, *, mode: str, target: Path) -> None:
|
||||
self.emit("task_start", None, {"mode": mode, "target": str(target)})
|
||||
|
||||
def finish(self, *, status: str) -> None:
|
||||
self.emit("task_end", None, {"status": status})
|
||||
|
||||
def run_stage(self, name: str, callback: Callable[[], Any]) -> Any:
|
||||
self.emit("stage_start", name, {})
|
||||
try:
|
||||
result = callback()
|
||||
except Exception as exc:
|
||||
self.emit("stage_end", name, {"status": "failed", "error": f"{type(exc).__name__}: {exc}"})
|
||||
raise
|
||||
self.emit("stage_end", name, {"status": "succeeded"})
|
||||
return result
|
||||
|
||||
def emit(self, event_type: EventType, stage: str | None, payload: dict[str, Any]) -> None:
|
||||
event = HarnessEvent(
|
||||
event_type=event_type,
|
||||
scan_id=self.scan_id,
|
||||
stage=stage,
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
payload=payload,
|
||||
)
|
||||
self.trace_writer.append(event)
|
||||
for processor in self.processors:
|
||||
if event_type in processor.spec.handles:
|
||||
self._run_processor(processor, event)
|
||||
|
||||
def _run_processor(self, processor: Processor, event: HarnessEvent) -> None:
|
||||
before = set(self.state)
|
||||
missing_reads = [key for key in processor.spec.reads if key not in self.state]
|
||||
if missing_reads:
|
||||
self._contract_violation(processor.spec.name, f"missing reads: {', '.join(missing_reads)}")
|
||||
return
|
||||
|
||||
updates = processor.handle(event, dict(self.state)) or {}
|
||||
if not isinstance(updates, dict):
|
||||
self._contract_violation(processor.spec.name, "processor returned a non-dict update")
|
||||
return
|
||||
undeclared = sorted(set(updates) - set(processor.spec.writes))
|
||||
if undeclared:
|
||||
self._contract_violation(processor.spec.name, f"undeclared writes: {', '.join(undeclared)}")
|
||||
return
|
||||
self.state.update(updates)
|
||||
changed = sorted(set(self.state) - before | set(updates))
|
||||
self.trace_writer.append(
|
||||
HarnessEvent(
|
||||
event_type=event.event_type,
|
||||
scan_id=self.scan_id,
|
||||
stage=event.stage,
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
payload={"processor": processor.spec.name, "changed_state": changed},
|
||||
)
|
||||
)
|
||||
|
||||
def _contract_violation(self, processor_name: str, reason: str) -> None:
|
||||
message = f"processor {processor_name} contract violation: {reason}"
|
||||
if self.contract_mode == "strict":
|
||||
raise HarnessContractError(message)
|
||||
self.state.setdefault("degradations", []).append(message)
|
||||
self.trace_writer.append(
|
||||
HarnessEvent(
|
||||
event_type="stage_end",
|
||||
scan_id=self.scan_id,
|
||||
stage="processor_contract",
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
payload={"status": "degraded", "reason": message},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def write_harness_config(
|
||||
*,
|
||||
config: ResolvedConfig,
|
||||
processors: list[Processor],
|
||||
contract_mode: ContractMode,
|
||||
path: Path,
|
||||
prompts: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
prompts = prompts or {}
|
||||
serialized = SerializedHarnessConfig(
|
||||
contract_mode=contract_mode,
|
||||
model_roles={
|
||||
"ranker": config.models.ranker,
|
||||
"hunter": config.models.hunter,
|
||||
"verifier": config.models.verifier,
|
||||
"patcher": config.models.patcher,
|
||||
},
|
||||
processor_versions={processor.spec.name: processor.spec.version for processor in processors},
|
||||
prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())},
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(asdict(serialized), indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _hash_text(text: str) -> str:
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
@@ -0,0 +1,455 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from .preprocess import FileTarget
|
||||
|
||||
MappingKind = Literal["semgrep", "codeql", "differential", "sharp_edge"]
|
||||
AccessLevel = Literal[
|
||||
"public",
|
||||
"authenticated",
|
||||
"role-restricted",
|
||||
"contract-only/callback",
|
||||
"local-only",
|
||||
"review-required",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StaticHint:
|
||||
analyzer: str
|
||||
rule_id: str
|
||||
title: str
|
||||
severity: str
|
||||
path: str
|
||||
start_line: int | None
|
||||
message: str
|
||||
fingerprint: str | None
|
||||
provenance: str
|
||||
evidence_candidate: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MappingTaskRecord:
|
||||
kind: MappingKind
|
||||
path: str
|
||||
rule_id: str
|
||||
task: str
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DifferentialMappingRecord:
|
||||
path: str
|
||||
change_kind: str
|
||||
trust_boundary: str | None
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SharpEdgeRecord:
|
||||
path: str
|
||||
category: str
|
||||
surface: str
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntryPointRecord:
|
||||
path: str
|
||||
line: int | None
|
||||
end_line: int | None
|
||||
function_name: str | None
|
||||
name: str
|
||||
kind: str
|
||||
access_level: AccessLevel
|
||||
trust_boundary: str
|
||||
access_evidence: list[str]
|
||||
conditions: list[str]
|
||||
provenance: str
|
||||
rationale: str
|
||||
|
||||
|
||||
def ingest_sarif(path: Path) -> list[StaticHint]:
|
||||
payload = json.loads(path.read_text())
|
||||
hints: list[StaticHint] = []
|
||||
for run in _items(payload.get("runs")):
|
||||
tool = run.get("tool") if isinstance(run, dict) else {}
|
||||
driver = tool.get("driver") if isinstance(tool, dict) else {}
|
||||
analyzer = _normalise_analyzer(str(driver.get("name", "sarif"))) if isinstance(driver, dict) else "sarif"
|
||||
rules = _rules_by_id(driver.get("rules", []) if isinstance(driver, dict) else [])
|
||||
for result in _items(run.get("results") if isinstance(run, dict) else []):
|
||||
hint = _hint_from_result(result, analyzer, rules)
|
||||
if hint is not None:
|
||||
hints.append(hint)
|
||||
return sorted(hints, key=lambda item: (item.path, item.start_line or 0, item.rule_id))
|
||||
|
||||
|
||||
def attach_static_hints(targets: list[FileTarget], hints: list[StaticHint]) -> list[FileTarget]:
|
||||
hints_by_path: dict[str, list[dict[str, object]]] = {}
|
||||
for hint in hints:
|
||||
hints_by_path.setdefault(hint.path, []).append(asdict(hint))
|
||||
return [replace(target, static_hints=hints_by_path.get(target.path, target.static_hints)) for target in targets]
|
||||
|
||||
|
||||
def analyze_entry_points(root: Path, targets: list[FileTarget]) -> list[EntryPointRecord]:
|
||||
records: list[EntryPointRecord] = []
|
||||
for target in targets:
|
||||
path = root / target.path
|
||||
text = path.read_text(errors="ignore")
|
||||
records.extend(_language_entry_points(target, text))
|
||||
records.extend(_tag_entry_points(target))
|
||||
return sorted(records, key=lambda item: (item.path, item.line is None, item.line or 0, item.name))
|
||||
|
||||
|
||||
def attach_reachability_hints(targets: list[FileTarget], entry_points: list[EntryPointRecord]) -> list[FileTarget]:
|
||||
hints_by_path: dict[str, list[dict[str, object]]] = {}
|
||||
for entry_point in entry_points:
|
||||
hints_by_path.setdefault(entry_point.path, []).append(asdict(entry_point))
|
||||
return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets]
|
||||
|
||||
|
||||
def verifier_evidence_candidates(hints: list[StaticHint], *, path: str | None = None) -> list[dict[str, object]]:
|
||||
selected = [hint for hint in hints if hint.evidence_candidate and (path is None or hint.path == path)]
|
||||
return [
|
||||
{
|
||||
"source": hint.analyzer,
|
||||
"rule_id": hint.rule_id,
|
||||
"path": hint.path,
|
||||
"line": hint.start_line,
|
||||
"severity": hint.severity,
|
||||
"message": hint.message,
|
||||
"provenance": hint.provenance,
|
||||
}
|
||||
for hint in selected
|
||||
]
|
||||
|
||||
|
||||
def semgrep_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
|
||||
return [
|
||||
MappingTaskRecord(
|
||||
kind="semgrep",
|
||||
path=hint.path,
|
||||
rule_id=hint.rule_id,
|
||||
task="pattern_variant_review",
|
||||
rationale="Use Semgrep evidence to check variants and rule precision before verifier promotion.",
|
||||
)
|
||||
for hint in hints
|
||||
if hint.analyzer == "semgrep"
|
||||
]
|
||||
|
||||
|
||||
def codeql_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
|
||||
return [
|
||||
MappingTaskRecord(
|
||||
kind="codeql",
|
||||
path=hint.path,
|
||||
rule_id=hint.rule_id,
|
||||
task="source_sink_sanitizer_path_review",
|
||||
rationale="Use CodeQL evidence to inspect source, sink, sanitizer, and path explanation.",
|
||||
)
|
||||
for hint in hints
|
||||
if hint.analyzer == "codeql"
|
||||
]
|
||||
|
||||
|
||||
def differential_mapping_record(path: str, *, change_kind: str, trust_boundary: str | None, rationale: str) -> DifferentialMappingRecord:
|
||||
return DifferentialMappingRecord(path=path, change_kind=change_kind, trust_boundary=trust_boundary, rationale=rationale)
|
||||
|
||||
|
||||
def sharp_edge_record(path: str, *, category: str, surface: str, rationale: str) -> SharpEdgeRecord:
|
||||
return SharpEdgeRecord(path=path, category=category, surface=surface, rationale=rationale)
|
||||
|
||||
|
||||
def write_static_hints(hints: list[StaticHint], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def write_entry_points(entry_points: list[EntryPointRecord], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"entry_points": [asdict(entry_point) for entry_point in entry_points]}, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _language_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
|
||||
if target.language == "python":
|
||||
return _python_entry_points(target, text)
|
||||
if target.language in {"javascript", "typescript"}:
|
||||
return _js_entry_points(target, text)
|
||||
if target.language in {"c", "cpp"}:
|
||||
return _c_entry_points(target, text)
|
||||
if target.language == "solidity":
|
||||
return _solidity_entry_points(target, text)
|
||||
return []
|
||||
|
||||
|
||||
def _python_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
|
||||
records: list[EntryPointRecord] = []
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
tree = None
|
||||
if tree is not None:
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
decorators = [_safe_unparse(decorator) for decorator in node.decorator_list]
|
||||
route = next((decorator for decorator in decorators if _looks_like_route_decorator(decorator)), None)
|
||||
if route is not None:
|
||||
access, evidence = _python_route_access(decorators)
|
||||
conditions = _python_conditions(node, decorators)
|
||||
records.append(
|
||||
_entry(
|
||||
target,
|
||||
node.lineno,
|
||||
getattr(node, "end_lineno", node.lineno),
|
||||
node.name,
|
||||
node.name,
|
||||
"route",
|
||||
access,
|
||||
"http_request",
|
||||
evidence or [route],
|
||||
conditions,
|
||||
)
|
||||
)
|
||||
if any("click.command" in decorator or ".command" in decorator for decorator in decorators):
|
||||
records.append(
|
||||
_entry(
|
||||
target,
|
||||
node.lineno,
|
||||
getattr(node, "end_lineno", node.lineno),
|
||||
node.name,
|
||||
node.name,
|
||||
"cli",
|
||||
"local-only",
|
||||
"process_invocation",
|
||||
decorators,
|
||||
_python_conditions(node, decorators),
|
||||
)
|
||||
)
|
||||
for number, line in enumerate(text.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("if __name__") and "__main__" in stripped:
|
||||
records.append(_entry(target, number, number, "__main__", "__main__", "cli", "local-only", "process_invocation", [stripped], []))
|
||||
if "argparse." in stripped or "click.command" in stripped:
|
||||
records.append(_entry(target, number, number, None, "cli_parser", "cli", "local-only", "process_invocation", [stripped], []))
|
||||
return records
|
||||
|
||||
|
||||
def _js_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
|
||||
records: list[EntryPointRecord] = []
|
||||
route_tokens = (".get(", ".post(", ".put(", ".delete(", ".patch(", ".use(")
|
||||
for number, line in enumerate(text.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if any(token in stripped for token in route_tokens) and any(prefix in stripped for prefix in ("app", "router", "server")):
|
||||
records.append(_entry(target, number, number, _js_handler_name(stripped), "http_handler", "route", "public", "http_request", [stripped], _line_conditions(stripped)))
|
||||
if "process.argv" in stripped:
|
||||
records.append(_entry(target, number, number, None, "process.argv", "cli", "local-only", "process_invocation", [stripped], _line_conditions(stripped)))
|
||||
return records
|
||||
|
||||
|
||||
def _c_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
|
||||
records: list[EntryPointRecord] = []
|
||||
for number, line in enumerate(text.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if " main(" in f" {stripped}" or stripped.startswith("main("):
|
||||
records.append(_entry(target, number, number, "main", "main", "cli", "local-only", "process_invocation", [stripped], _line_conditions(stripped)))
|
||||
if "LLVMFuzzerTestOneInput" in stripped:
|
||||
records.append(_entry(target, number, number, "LLVMFuzzerTestOneInput", "LLVMFuzzerTestOneInput", "fuzz", "public", "fuzzer_input", [stripped], _line_conditions(stripped)))
|
||||
return records
|
||||
|
||||
|
||||
def _solidity_entry_points(target: FileTarget, text: str) -> list[EntryPointRecord]:
|
||||
records: list[EntryPointRecord] = []
|
||||
for number, line in enumerate(text.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("function ") or not any(token in stripped for token in (" public", " external")):
|
||||
continue
|
||||
if " view" in stripped or " pure" in stripped:
|
||||
continue
|
||||
evidence = [token for token in ("onlyOwner", "onlyRole", "requiresAuth") if token in stripped]
|
||||
conditions = [token for token in ("whenNotPaused", "whenPaused", "featureEnabled") if token in stripped]
|
||||
access = "role-restricted" if evidence else "public"
|
||||
name = stripped.removeprefix("function ").split("(", 1)[0]
|
||||
if "callback" in name.lower() or name.startswith("onERC"):
|
||||
access = "contract-only/callback"
|
||||
evidence.append("callback naming convention")
|
||||
records.append(_entry(target, number, number, name, name, "smart_contract_state_change", access, "contract_call", evidence or ["external/public function"], conditions))
|
||||
return records
|
||||
|
||||
|
||||
def _tag_entry_points(target: FileTarget) -> list[EntryPointRecord]:
|
||||
records: list[EntryPointRecord] = []
|
||||
tags = set(target.tags)
|
||||
if "parser" in tags or "deserialization" in tags:
|
||||
records.append(_entry(target, None, None, None, "parser_input", "parser", "public", "attacker_controlled_input", ["parser/deserialization file tag"], []))
|
||||
if "network_entry" in tags:
|
||||
records.append(_entry(target, None, None, None, "network_surface", "route", "public", "network_request", ["network entry file tag"], []))
|
||||
if "auth_boundary" in tags:
|
||||
records.append(_entry(target, None, None, None, "auth_boundary", "privileged_surface", "authenticated", "identity_boundary", ["auth boundary file tag"], []))
|
||||
if "syscall_entry" in tags or "filesystem_entry" in tags:
|
||||
records.append(_entry(target, None, None, None, "local_privileged_boundary", "privileged_surface", "local-only", "local_process", ["syscall/filesystem file tag"], []))
|
||||
return records
|
||||
|
||||
|
||||
def _entry(
|
||||
target: FileTarget,
|
||||
line: int | None,
|
||||
end_line: int | None,
|
||||
function_name: str | None,
|
||||
name: str,
|
||||
kind: str,
|
||||
access_level: AccessLevel,
|
||||
trust_boundary: str,
|
||||
access_evidence: list[str],
|
||||
conditions: list[str],
|
||||
) -> EntryPointRecord:
|
||||
return EntryPointRecord(
|
||||
path=target.path,
|
||||
line=line,
|
||||
end_line=end_line,
|
||||
function_name=function_name,
|
||||
name=name,
|
||||
kind=kind,
|
||||
access_level=access_level,
|
||||
trust_boundary=trust_boundary,
|
||||
access_evidence=access_evidence,
|
||||
conditions=conditions,
|
||||
provenance=f"entrypoint:{kind}:{access_level}",
|
||||
rationale=f"{kind} surface classified as {access_level} at {trust_boundary}",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_route_decorator(line: str) -> bool:
|
||||
return any(token in line for token in ("route", ".get", ".post", ".put", ".delete", ".patch"))
|
||||
|
||||
|
||||
def _safe_unparse(node: ast.AST) -> str:
|
||||
try:
|
||||
return ast.unparse(node)
|
||||
except (AttributeError, ValueError):
|
||||
return ""
|
||||
|
||||
|
||||
def _python_route_access(decorators: list[str]) -> tuple[AccessLevel, list[str]]:
|
||||
role_tokens = ("role", "permission", "admin", "owner")
|
||||
auth_tokens = ("login_required", "authenticated", "auth", "jwt_required")
|
||||
access_decorators = [decorator for decorator in decorators if not _looks_like_route_decorator(decorator)]
|
||||
evidence = [decorator for decorator in access_decorators if any(token in decorator.lower() for token in role_tokens + auth_tokens)]
|
||||
lowered = "\n".join(evidence).lower()
|
||||
if any(token in lowered for token in role_tokens):
|
||||
return "role-restricted", evidence
|
||||
if any(token in lowered for token in auth_tokens):
|
||||
return "authenticated", evidence
|
||||
return "public", evidence
|
||||
|
||||
|
||||
def _python_conditions(node: ast.FunctionDef | ast.AsyncFunctionDef, decorators: list[str]) -> list[str]:
|
||||
conditions = [decorator for decorator in decorators if _looks_conditional(decorator)]
|
||||
for child in ast.walk(node):
|
||||
if isinstance(child, ast.If):
|
||||
test = _safe_unparse(child.test)
|
||||
if _looks_conditional(test):
|
||||
conditions.append(test)
|
||||
return sorted(set(conditions))
|
||||
|
||||
|
||||
def _line_conditions(line: str) -> list[str]:
|
||||
return [line] if _looks_conditional(line) else []
|
||||
|
||||
|
||||
def _looks_conditional(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
return any(token in lowered for token in ("feature", "flag", "toggle", "enabled", "disabled", "experiment", "beta", "rollout", "paused"))
|
||||
|
||||
|
||||
def _js_handler_name(line: str) -> str | None:
|
||||
if "function " in line:
|
||||
return line.split("function ", 1)[1].split("(", 1)[0].strip() or None
|
||||
if "=>" in line:
|
||||
return "arrow_handler"
|
||||
return None
|
||||
|
||||
|
||||
def _hint_from_result(result: object, analyzer: str, rules: dict[str, dict[str, object]]) -> StaticHint | None:
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
rule_id = str(result.get("ruleId", "unknown"))
|
||||
location = _primary_location(result)
|
||||
if location is None:
|
||||
return None
|
||||
artifact = location.get("physicalLocation", {}).get("artifactLocation", {})
|
||||
region = location.get("physicalLocation", {}).get("region", {})
|
||||
uri = artifact.get("uri") if isinstance(artifact, dict) else None
|
||||
if not isinstance(uri, str) or not uri:
|
||||
return None
|
||||
rule = rules.get(rule_id, {})
|
||||
message = result.get("message", {})
|
||||
message_text = message.get("text") if isinstance(message, dict) else None
|
||||
return StaticHint(
|
||||
analyzer=analyzer,
|
||||
rule_id=rule_id,
|
||||
title=str(rule.get("name") or rule.get("shortDescription") or rule_id),
|
||||
severity=_severity(result, rule),
|
||||
path=uri.lstrip("./"),
|
||||
start_line=region.get("startLine") if isinstance(region.get("startLine"), int) else None,
|
||||
message=message_text if isinstance(message_text, str) else rule_id,
|
||||
fingerprint=_fingerprint(result),
|
||||
provenance=f"sarif:{analyzer}:{rule_id}",
|
||||
)
|
||||
|
||||
|
||||
def _primary_location(result: dict[str, object]) -> dict[str, Any] | None:
|
||||
locations = result.get("locations")
|
||||
if not isinstance(locations, list) or not locations or not isinstance(locations[0], dict):
|
||||
return None
|
||||
return locations[0]
|
||||
|
||||
|
||||
def _rules_by_id(rules: object) -> dict[str, dict[str, object]]:
|
||||
by_id: dict[str, dict[str, object]] = {}
|
||||
for rule in _items(rules):
|
||||
rule_id = rule.get("id") if isinstance(rule, dict) else None
|
||||
if isinstance(rule_id, str):
|
||||
by_id[rule_id] = rule
|
||||
return by_id
|
||||
|
||||
|
||||
def _severity(result: dict[str, object], rule: dict[str, object]) -> str:
|
||||
level = result.get("level")
|
||||
if level == "error":
|
||||
return "high"
|
||||
if level == "warning":
|
||||
return "medium"
|
||||
properties = rule.get("properties")
|
||||
if isinstance(properties, dict) and "security-severity" in properties:
|
||||
try:
|
||||
return "high" if float(properties["security-severity"]) >= 7.0 else "medium"
|
||||
except (TypeError, ValueError):
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def _fingerprint(result: dict[str, object]) -> str | None:
|
||||
for key in ("partialFingerprints", "fingerprints"):
|
||||
value = result.get(key)
|
||||
if isinstance(value, dict) and value:
|
||||
first = next(iter(value.values()))
|
||||
return str(first)
|
||||
return None
|
||||
|
||||
|
||||
def _normalise_analyzer(name: str) -> str:
|
||||
lowered = name.lower()
|
||||
if "semgrep" in lowered:
|
||||
return "semgrep"
|
||||
if "codeql" in lowered:
|
||||
return "codeql"
|
||||
return lowered.replace(" ", "_") or "sarif"
|
||||
|
||||
|
||||
def _items(value: object) -> list[object]:
|
||||
return value if isinstance(value, list) else []
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import fnmatch
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import asdict, dataclass
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ LANGUAGE_BY_EXTENSION = {
|
||||
".php": "php",
|
||||
".cs": "csharp",
|
||||
".swift": "swift",
|
||||
".sol": "solidity",
|
||||
}
|
||||
|
||||
MEMORY_UNSAFE_LANGUAGES = {"c", "cpp"}
|
||||
@@ -50,12 +51,22 @@ class FileTarget:
|
||||
loc: int
|
||||
tags: list[str]
|
||||
has_fuzz_entry_point: bool
|
||||
static_hints: list[dict[str, object]] = field(default_factory=list)
|
||||
reachability_hints: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
|
||||
def preprocess_repository(root: Path, output_path: Path | None = None) -> tuple[RepoSnapshot, list[FileTarget]]:
|
||||
def preprocess_repository(
|
||||
root: Path,
|
||||
output_path: Path | None = None,
|
||||
static_hints: list[object] | None = None,
|
||||
) -> tuple[RepoSnapshot, list[FileTarget]]:
|
||||
resolved = root.resolve()
|
||||
files = enumerate_source_files(resolved)
|
||||
targets = [build_file_target(resolved, path) for path in files]
|
||||
if static_hints:
|
||||
from .mapping import attach_static_hints
|
||||
|
||||
targets = attach_static_hints(targets, static_hints) # type: ignore[arg-type]
|
||||
languages: dict[str, int] = {}
|
||||
for target in targets:
|
||||
languages[target.language] = languages.get(target.language, 0) + 1
|
||||
@@ -68,13 +79,17 @@ def preprocess_repository(root: Path, output_path: Path | None = None) -> tuple[
|
||||
)
|
||||
|
||||
if output_path is not None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"snapshot": asdict(snapshot), "file_targets": [asdict(target) for target in targets]}
|
||||
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
write_preprocess_artifact(snapshot, targets, output_path)
|
||||
|
||||
return snapshot, targets
|
||||
|
||||
|
||||
def write_preprocess_artifact(snapshot: RepoSnapshot, targets: list[FileTarget], output_path: Path) -> None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"snapshot": asdict(snapshot), "file_targets": [asdict(target) for target in targets]}
|
||||
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def enumerate_source_files(root: Path) -> list[Path]:
|
||||
patterns = _load_ignore_patterns(root)
|
||||
paths: list[Path] = []
|
||||
@@ -97,6 +112,8 @@ def build_file_target(root: Path, path: Path) -> FileTarget:
|
||||
loc=count_loc(text),
|
||||
tags=tags,
|
||||
has_fuzz_entry_point="LLVMFuzzerTestOneInput" in text,
|
||||
static_hints=[],
|
||||
reachability_hints=[],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -105,6 +105,10 @@ def heuristic_rank(target: FileTarget) -> RankingScore:
|
||||
if target.loc >= 500:
|
||||
influence = max(influence, 3)
|
||||
boosts.append("large_file")
|
||||
reachability_boost = _reachability_from_hints(target.reachability_hints)
|
||||
if reachability_boost:
|
||||
reachability = max(reachability, reachability_boost)
|
||||
boosts.append("entry_point_reachable")
|
||||
|
||||
return RankingScore(
|
||||
path=target.path,
|
||||
@@ -212,4 +216,23 @@ def _target_prompt_payload(target: FileTarget) -> dict[str, object]:
|
||||
"loc": target.loc,
|
||||
"tags": target.tags,
|
||||
"has_fuzz_entry_point": target.has_fuzz_entry_point,
|
||||
"reachability_hints": target.reachability_hints,
|
||||
}
|
||||
|
||||
|
||||
def _reachability_from_hints(hints: list[dict[str, object]]) -> int:
|
||||
score = 0
|
||||
for hint in hints:
|
||||
access_level = hint.get("access_level")
|
||||
kind = hint.get("kind")
|
||||
if access_level == "public":
|
||||
score = max(score, 5 if kind in {"route", "parser", "fuzz"} else 4)
|
||||
elif access_level in {"authenticated", "contract-only/callback"}:
|
||||
score = max(score, 4)
|
||||
elif access_level == "role-restricted":
|
||||
score = max(score, 3)
|
||||
elif access_level == "review-required":
|
||||
score = max(score, 2)
|
||||
elif access_level == "local-only":
|
||||
score = max(score, 2)
|
||||
return score
|
||||
|
||||
@@ -18,9 +18,12 @@ def write_markdown_report(findings: list[StaticFinding], path: Path) -> None:
|
||||
f"- ID: `{finding.finding_id}`",
|
||||
f"- Path: `{finding.path}`",
|
||||
f"- Line: `{finding.line}`",
|
||||
f"- Function: `{finding.function_name or 'unknown'}`",
|
||||
f"- Severity: `{finding.severity}`",
|
||||
f"- Confidence: `{finding.confidence}`",
|
||||
f"- Evidence level: `{finding.evidence_level}`",
|
||||
f"- Reachability: `{finding.reachability_status}`",
|
||||
f"- Conditions: `{', '.join(finding.reachability_conditions) or 'none'}`",
|
||||
f"- Ranking priority: `{finding.ranking_priority}`",
|
||||
f"- Tags: `{', '.join(finding.tags)}`",
|
||||
"",
|
||||
|
||||
Reference in New Issue
Block a user