mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
f5a904b7f2
commit
f63703a604
@@ -47,16 +47,6 @@ class FalsePositiveLearning:
|
||||
demotion: float = 0.5
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RankingMetrics:
|
||||
tier: str
|
||||
verified_findings: int
|
||||
false_positives: int
|
||||
time_to_verification_seconds: float
|
||||
missed_fixture_vulnerabilities: int
|
||||
false_positive_reasons: dict[str, int]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RankingCalibration:
|
||||
path: str
|
||||
@@ -105,38 +95,6 @@ def learning_from_finding_outcome(
|
||||
)
|
||||
|
||||
|
||||
def compute_ranking_metrics(events: list[dict[str, object]], *, tier: str) -> RankingMetrics:
|
||||
verified = 0
|
||||
false_positives = 0
|
||||
reason_counts: dict[str, int] = {}
|
||||
verification_times: list[float] = []
|
||||
missed = 0
|
||||
for event in events:
|
||||
if event.get("tier") != tier:
|
||||
continue
|
||||
outcome = event.get("outcome")
|
||||
if outcome == FindingOutcome.VERIFIED:
|
||||
verified += 1
|
||||
if outcome in _FALSE_POSITIVE_OUTCOMES:
|
||||
false_positives += 1
|
||||
reason = str(event.get("reason", FalsePositiveReason.UNVERIFIED))
|
||||
reason_counts[reason] = reason_counts.get(reason, 0) + 1
|
||||
duration = event.get("time_to_verification_seconds")
|
||||
if isinstance(duration, int | float) and not isinstance(duration, bool):
|
||||
verification_times.append(float(duration))
|
||||
missed_value = event.get("missed_fixture_vulnerabilities", 0)
|
||||
if isinstance(missed_value, int) and not isinstance(missed_value, bool):
|
||||
missed += missed_value
|
||||
return RankingMetrics(
|
||||
tier=tier,
|
||||
verified_findings=verified,
|
||||
false_positives=false_positives,
|
||||
time_to_verification_seconds=round(sum(verification_times) / len(verification_times), 2) if verification_times else 0.0,
|
||||
missed_fixture_vulnerabilities=missed,
|
||||
false_positive_reasons=dict(sorted(reason_counts.items())),
|
||||
)
|
||||
|
||||
|
||||
def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearning]) -> RankingCalibration:
|
||||
applicable = [learning for learning in learnings if _learning_applies(ranking.path, learning)]
|
||||
calibrated = ranking.priority
|
||||
|
||||
@@ -548,7 +548,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
config=config,
|
||||
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets),
|
||||
)
|
||||
write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json")
|
||||
write_harness_config(config=config, path=run.root / "harness.json")
|
||||
runtime.start(mode=mode, target=run.target)
|
||||
policy = runtime.run_stage("policy_load", load_policy)
|
||||
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
||||
|
||||
@@ -99,17 +99,6 @@ def uncovered_languages(
|
||||
return sorted(counts.items())
|
||||
|
||||
|
||||
def build_quick_hunter_prompt(target: FileTarget, source_excerpt: str) -> str:
|
||||
return (
|
||||
"Review this file for security-relevant issues. Return structured findings only when evidence exists.\n"
|
||||
f"Path: {target.path}\n"
|
||||
f"Language: {target.language}\n"
|
||||
f"Tags: {', '.join(target.tags) or 'none'}\n"
|
||||
"Source excerpt:\n"
|
||||
f"{source_excerpt[:4000]}"
|
||||
)
|
||||
|
||||
|
||||
def write_findings(findings: list[StaticFinding], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"findings": [asdict(finding) for finding in findings]}
|
||||
|
||||
@@ -91,10 +91,6 @@ def fusion_triggers(
|
||||
return sorted(triggers)
|
||||
|
||||
|
||||
def should_fuse(finding: StaticFinding, verification: VerificationResult | None = None, *, high_assurance: bool = False) -> bool:
|
||||
return bool(fusion_triggers(finding, verification, high_assurance=high_assurance))
|
||||
|
||||
|
||||
def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict:
|
||||
"""Vulnerability-leaning panel: steel-mans the vulnerability case."""
|
||||
reachable = finding.reachability_status == "reachable"
|
||||
@@ -213,5 +209,4 @@ __all__ = [
|
||||
"fuse_finding",
|
||||
"fuse_findings",
|
||||
"fusion_triggers",
|
||||
"should_fuse",
|
||||
]
|
||||
]
|
||||
|
||||
@@ -6,7 +6,7 @@ from collections.abc import Callable
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, Protocol
|
||||
from typing import Any, Literal
|
||||
|
||||
from .config import ResolvedConfig
|
||||
|
||||
@@ -21,11 +21,6 @@ EventType = Literal[
|
||||
"task_end",
|
||||
]
|
||||
|
||||
ContractMode = Literal["strict", "warn"]
|
||||
|
||||
|
||||
class HarnessContractError(ValueError):
|
||||
"""Raised when a processor violates its declared state contract."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -37,27 +32,9 @@ class HarnessEvent:
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessorSpec:
|
||||
name: str
|
||||
version: str
|
||||
handles: tuple[EventType, ...]
|
||||
reads: tuple[str, ...] = ()
|
||||
writes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class Processor(Protocol):
|
||||
spec: ProcessorSpec
|
||||
|
||||
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SerializedHarnessConfig:
|
||||
contract_mode: ContractMode
|
||||
model_roles: dict[str, str | None]
|
||||
processor_versions: dict[str, str]
|
||||
prompt_hashes: dict[str, str]
|
||||
|
||||
|
||||
@@ -84,14 +61,10 @@ class HarnessRuntime:
|
||||
scan_id: str,
|
||||
config: ResolvedConfig,
|
||||
trace_writer: HarnessTraceWriter,
|
||||
processors: list[Processor] | None = None,
|
||||
contract_mode: ContractMode = "strict",
|
||||
) -> None:
|
||||
self.scan_id = scan_id
|
||||
self.config = config
|
||||
self.trace_writer = trace_writer
|
||||
self.processors = processors or []
|
||||
self.contract_mode = contract_mode
|
||||
self.state: dict[str, Any] = {"degradations": []}
|
||||
|
||||
def start(self, *, mode: str, target: Path) -> None:
|
||||
@@ -119,71 +92,22 @@ class HarnessRuntime:
|
||||
payload=payload,
|
||||
)
|
||||
self.trace_writer.append(event)
|
||||
for processor in self.processors:
|
||||
if event_type in processor.spec.handles:
|
||||
self._run_processor(processor, event)
|
||||
|
||||
def _run_processor(self, processor: Processor, event: HarnessEvent) -> None:
|
||||
before = set(self.state)
|
||||
missing_reads = [key for key in processor.spec.reads if key not in self.state]
|
||||
if missing_reads:
|
||||
self._contract_violation(processor.spec.name, f"missing reads: {', '.join(missing_reads)}")
|
||||
return
|
||||
|
||||
updates = processor.handle(event, dict(self.state)) or {}
|
||||
if not isinstance(updates, dict):
|
||||
self._contract_violation(processor.spec.name, "processor returned a non-dict update")
|
||||
return
|
||||
undeclared = sorted(set(updates) - set(processor.spec.writes))
|
||||
if undeclared:
|
||||
self._contract_violation(processor.spec.name, f"undeclared writes: {', '.join(undeclared)}")
|
||||
return
|
||||
self.state.update(updates)
|
||||
changed = sorted(set(self.state) - before | set(updates))
|
||||
self.trace_writer.append(
|
||||
HarnessEvent(
|
||||
event_type=event.event_type,
|
||||
scan_id=self.scan_id,
|
||||
stage=event.stage,
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
payload={"processor": processor.spec.name, "changed_state": changed},
|
||||
)
|
||||
)
|
||||
|
||||
def _contract_violation(self, processor_name: str, reason: str) -> None:
|
||||
message = f"processor {processor_name} contract violation: {reason}"
|
||||
if self.contract_mode == "strict":
|
||||
raise HarnessContractError(message)
|
||||
self.state.setdefault("degradations", []).append(message)
|
||||
self.trace_writer.append(
|
||||
HarnessEvent(
|
||||
event_type="stage_end",
|
||||
scan_id=self.scan_id,
|
||||
stage="processor_contract",
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
payload={"status": "degraded", "reason": message},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def write_harness_config(
|
||||
*,
|
||||
config: ResolvedConfig,
|
||||
processors: list[Processor],
|
||||
contract_mode: ContractMode,
|
||||
path: Path,
|
||||
prompts: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
prompts = prompts or {}
|
||||
serialized = SerializedHarnessConfig(
|
||||
contract_mode=contract_mode,
|
||||
model_roles={
|
||||
"ranker": config.models.ranker,
|
||||
"hunter": config.models.hunter,
|
||||
"verifier": None, # schema kept byte for byte; LLM verification runs on the plane
|
||||
"patcher": None, # schema kept; `[models] patcher` was retired 2026-10-02 (never read)
|
||||
},
|
||||
processor_versions={processor.spec.name: processor.spec.version for processor in processors},
|
||||
prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())},
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -2,10 +2,10 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from .preprocess import FileTarget, enumerate_source_files
|
||||
from .preprocess import enumerate_source_files
|
||||
|
||||
|
||||
class PathEscapesRepo(ValueError):
|
||||
@@ -80,25 +80,9 @@ def grep_repo(root: Path, pattern: str, *, max_matches: int) -> list[dict[str, o
|
||||
return matches
|
||||
|
||||
|
||||
def find_refs(root: Path, symbol: str, mapping_index: object) -> list[dict[str, object]]:
|
||||
def find_refs(root: Path, symbol: str) -> list[dict[str, object]]:
|
||||
resolved_root = root.resolve()
|
||||
refs: list[dict[str, object]] = []
|
||||
for record in _iter_mapping_records(mapping_index):
|
||||
if not _record_matches_symbol(record, symbol):
|
||||
continue
|
||||
raw_path = _record_path(record)
|
||||
if raw_path is None:
|
||||
continue
|
||||
clamped = clamp_repo_path(resolved_root, raw_path)
|
||||
name = _record_value(record, "name", "function_name", "symbol")
|
||||
refs.append(
|
||||
{
|
||||
"path": clamped.relative_to(resolved_root).as_posix(),
|
||||
"name": str(name) if name is not None else symbol,
|
||||
"line": _as_line(_record_value(record, "line")),
|
||||
"source": "mapping",
|
||||
}
|
||||
)
|
||||
if symbol:
|
||||
compiled = re.compile(rf"\b{re.escape(symbol)}\b")
|
||||
for relative, lines in _source_lines(resolved_root):
|
||||
@@ -113,221 +97,3 @@ def _iter_clamped_source_files(root: Path) -> Iterator[tuple[str, Path]]:
|
||||
for path in enumerate_source_files(resolved_root):
|
||||
relative = path.relative_to(resolved_root).as_posix()
|
||||
yield relative, clamp_repo_path(resolved_root, relative)
|
||||
|
||||
|
||||
def _iter_mapping_records(mapping_index: object) -> Iterable[object]:
|
||||
if mapping_index is None:
|
||||
return ()
|
||||
if isinstance(mapping_index, Mapping):
|
||||
return (mapping_index,)
|
||||
if isinstance(mapping_index, (str, bytes, bytearray)):
|
||||
return ()
|
||||
if isinstance(mapping_index, Iterable):
|
||||
return mapping_index
|
||||
return (mapping_index,)
|
||||
|
||||
|
||||
def _record_matches_symbol(record: object, symbol: str) -> bool:
|
||||
for field in ("name", "function_name", "symbol"):
|
||||
value = _record_value(record, field)
|
||||
if value is not None and str(value) == symbol:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _record_path(record: object) -> str | None:
|
||||
value = record.get("path") if isinstance(record, Mapping) else getattr(record, "path", None)
|
||||
if value is None:
|
||||
return None
|
||||
return str(value)
|
||||
|
||||
|
||||
def _record_value(record: object, *names: str) -> object | None:
|
||||
for name in names:
|
||||
raw: object | None
|
||||
if isinstance(record, Mapping):
|
||||
if name not in record:
|
||||
continue
|
||||
raw = record[name]
|
||||
else:
|
||||
raw = getattr(record, name, None)
|
||||
if raw is not None:
|
||||
return raw
|
||||
return None
|
||||
|
||||
|
||||
def _as_line(value: object) -> int | None:
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
# --- curated structural tools (learning-harness, Req 6.4, 6.5) --------------------------------------
|
||||
# A detector reads code with `read_file` and `read_definition`; the three list tools below answer with
|
||||
# identifiers and line spans only, so they can never become a second way to dump the repository. Every
|
||||
# path a caller supplies is clamped to the scan root first. No caching: a detector calls these a handful
|
||||
# of times per region, and a stale view of a tree that a round may have rewritten is worse than a reparse.
|
||||
|
||||
|
||||
def read_definition(root: Path, symbol: str, *, max_chars: int, path: str | None = None) -> dict[str, object] | None:
|
||||
"""The definition of one named function: where it is, how far it runs, and its text. None when absent."""
|
||||
resolved_root = root.resolve()
|
||||
scope = clamp_repo_path(resolved_root, path) if path is not None else None
|
||||
for relative, target in _iter_clamped_source_files(resolved_root):
|
||||
if scope is not None and target != scope:
|
||||
continue
|
||||
text = target.read_text(errors="ignore")
|
||||
span = _named_span(relative, text, symbol)
|
||||
if span is None:
|
||||
continue
|
||||
lines = text.splitlines()
|
||||
start, end = span
|
||||
while start > 1 and lines[start - 2].lstrip().startswith("@"):
|
||||
start -= 1 # a handler's decorators are part of what the detector must see
|
||||
body = "\n".join(lines[start - 1 : end])
|
||||
return {"path": relative, "name": symbol, "start_line": start, "end_line": end, "text": body[: max(max_chars, 0)]}
|
||||
return None
|
||||
|
||||
|
||||
def entry_points(root: Path, *, path: str | None = None) -> list[dict[str, object]]:
|
||||
"""Handlers the mapper names, with their access classification and evidence. No source text."""
|
||||
from .mapping import analyze_entry_points
|
||||
|
||||
resolved_root = root.resolve()
|
||||
relative = _relative_or_none(resolved_root, path)
|
||||
targets = _repo_targets(resolved_root)
|
||||
rows: list[dict[str, object]] = []
|
||||
for record in analyze_entry_points(resolved_root, targets):
|
||||
if relative is not None and record.path != relative:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"path": record.path,
|
||||
"function": record.function_name,
|
||||
"name": record.name,
|
||||
"kind": record.kind,
|
||||
"line": record.line,
|
||||
"end_line": record.end_line,
|
||||
"access_level": record.access_level,
|
||||
"evidence": list(record.access_evidence),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def flows(root: Path, *, path: str, function: str) -> list[dict[str, object]]:
|
||||
"""Adjudicated source-to-sink records inside one function: which source, which sink, what the overlay said."""
|
||||
from .findings import quick_scan_findings
|
||||
from .rank import rank_targets
|
||||
from .semantic import adjudicate
|
||||
|
||||
resolved_root = root.resolve()
|
||||
relative = _relative_or_none(resolved_root, path)
|
||||
targets = _repo_targets(resolved_root)
|
||||
span = _span_in(resolved_root, relative, function)
|
||||
if span is None:
|
||||
return []
|
||||
findings = [item for item in quick_scan_findings(resolved_root, targets, rank_targets(targets), None) if item.status != "shadow"]
|
||||
rows: list[dict[str, object]] = []
|
||||
for record in adjudicate(root=resolved_root, targets=targets, findings=findings):
|
||||
if record.path != relative or record.line is None or not span[0] <= record.line <= span[1]:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"path": record.path,
|
||||
"function": function,
|
||||
"line": record.line,
|
||||
"sources": list(record.sources),
|
||||
"sinks": list(record.sinks),
|
||||
"disposition": record.disposition,
|
||||
"cwe": record.cwe,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def obligations(root: Path, *, path: str, function: str) -> list[dict[str, object]]:
|
||||
"""What one function owes and has not discharged: the operation, the missing discharger, why it is owed."""
|
||||
from .config import ObligationsConfig
|
||||
from .semantic.facts import FactLoadError, load_facts
|
||||
from .semantic.ir import parse_file
|
||||
from .semantic.obligations import check_obligations, load_obligation_facts
|
||||
from .semantic.obligations.dominance import OrderDominance
|
||||
|
||||
resolved_root = root.resolve()
|
||||
relative = _relative_or_none(resolved_root, path)
|
||||
targets = _repo_targets(resolved_root)
|
||||
try:
|
||||
flow_facts = load_facts()
|
||||
except FactLoadError:
|
||||
return []
|
||||
irs: dict[str, tuple[object, str]] = {}
|
||||
texts: dict[str, str] = {}
|
||||
for target in targets:
|
||||
try:
|
||||
text = (resolved_root / target.path).read_text(errors="ignore")
|
||||
except OSError:
|
||||
continue
|
||||
texts[target.path] = text
|
||||
irs[target.path] = (parse_file(target.path, text, target.language), text)
|
||||
from .mapping import analyze_entry_points
|
||||
|
||||
result = check_obligations(
|
||||
irs=irs, # type: ignore[arg-type]
|
||||
entries=analyze_entry_points(resolved_root, targets),
|
||||
facts=load_obligation_facts(),
|
||||
flow_facts=flow_facts,
|
||||
policy=None,
|
||||
paths=(),
|
||||
dominance=OrderDominance(texts=texts),
|
||||
store_shapes=(),
|
||||
min_siblings=ObligationsConfig().min_siblings,
|
||||
)
|
||||
return [
|
||||
{
|
||||
"path": finding.operation.path,
|
||||
"function": finding.operation.function,
|
||||
"line": finding.operation.line,
|
||||
"operation": finding.operation.kind,
|
||||
"resource": finding.operation.resource,
|
||||
"missing": finding.missing,
|
||||
"label": finding.label,
|
||||
"evidence": list(finding.evidence),
|
||||
}
|
||||
for finding in result.findings
|
||||
if (relative is None or finding.operation.path == relative) and finding.operation.function == function
|
||||
]
|
||||
|
||||
|
||||
def _repo_targets(root: Path) -> list[FileTarget]:
|
||||
from .preprocess import preprocess_repository
|
||||
|
||||
_, targets = preprocess_repository(root)
|
||||
return list(targets)
|
||||
|
||||
|
||||
def _relative_or_none(root: Path, path: str | None) -> str | None:
|
||||
if path is None:
|
||||
return None
|
||||
return clamp_repo_path(root, path).relative_to(root).as_posix()
|
||||
|
||||
|
||||
def _named_span(relative: str, text: str, symbol: str) -> tuple[int, int] | None:
|
||||
from .preprocess import detect_language
|
||||
from .semantic.functions import named_function_ranges, spans_named
|
||||
|
||||
ranges = named_function_ranges(relative, text, detect_language(Path(relative)))
|
||||
spans = spans_named(ranges or (), symbol)
|
||||
return spans[0] if spans else None
|
||||
|
||||
|
||||
def _span_in(root: Path, relative: str | None, function: str) -> tuple[int, int] | None:
|
||||
if relative is None:
|
||||
return None
|
||||
try:
|
||||
text = (root / relative).read_text(errors="ignore")
|
||||
except OSError:
|
||||
return None
|
||||
return _named_span(relative, text, function)
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
"""Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1).
|
||||
|
||||
No model. The same per-instrument builders serve two callers:
|
||||
|
||||
- the plane task ``features`` (:mod:`..plane.tasks.features`), which reads a case's delivered artifacts;
|
||||
- the host: :func:`build_for_scan` over a checkout's quick findings and an engine result, for ``scan`` and
|
||||
``pre-push`` (a whole-repository scan, or a base..head delta with :class:`Delta`).
|
||||
The plane task ``features`` (:mod:`..plane.tasks.features`) reads a case's delivered artifacts.
|
||||
|
||||
A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument
|
||||
emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the
|
||||
@@ -26,8 +22,8 @@ from typing import Any
|
||||
from ..model.taxonomy import load_families
|
||||
from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files
|
||||
from ..preprocess import build_file_target, detect_language
|
||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
|
||||
from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, infer_for_checkout, origin_of
|
||||
from ..ruleset import PatternRule
|
||||
from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, origin_of
|
||||
from .schema import (
|
||||
BY_NAME,
|
||||
LANGUAGES,
|
||||
@@ -62,12 +58,6 @@ NONE = Part("none")
|
||||
NOT_APPLICABLE = Part("not_applicable")
|
||||
|
||||
|
||||
def language_of(path: str) -> str:
|
||||
"""The closed language of a path (its extension), ``other`` outside :data:`.schema.LANGUAGES`."""
|
||||
language = detect_language(Path(path))
|
||||
return language if language in LANGUAGES else "other"
|
||||
|
||||
|
||||
def _clip(name: str, value: Any) -> Any:
|
||||
spec = BY_NAME[name]
|
||||
if value is None or spec.type != "int" or isinstance(value, bool):
|
||||
@@ -328,18 +318,6 @@ def function_of(lines: Sequence[str] | None, path: str, language: str, function:
|
||||
# --- the host path ---------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Delta:
|
||||
"""A base..head delta: the base checkout, the head-side changed line ranges per path (``git diff -U0``), and the
|
||||
base's quick findings and engine result, whose signals decide novelty."""
|
||||
|
||||
root: Path
|
||||
changed: Mapping[str, Sequence[tuple[int, int]]]
|
||||
findings: Sequence[Any] = ()
|
||||
engine_result: Mapping[str, Any] | None = None
|
||||
pin: str | None = None
|
||||
|
||||
|
||||
class SourceIndex:
|
||||
"""A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level
|
||||
caller graph (the repo-facts patterns) for the entry-point distance."""
|
||||
@@ -423,12 +401,6 @@ def _family_of(rule: PatternRule | None) -> str:
|
||||
return family.id if family is not None else UNKNOWN_FAMILY
|
||||
|
||||
|
||||
def _engine_state(result: Mapping[str, Any] | None) -> str:
|
||||
if result is None:
|
||||
return "none"
|
||||
return "ran" if int(result.get("questions") or 0) > 0 else "failed"
|
||||
|
||||
|
||||
def _signals(
|
||||
index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule]
|
||||
) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]:
|
||||
@@ -447,108 +419,6 @@ def _signals(
|
||||
return quick, engine
|
||||
|
||||
|
||||
def _changed_in(index: SourceIndex, changed: Mapping[str, Sequence[tuple[int, int]]]) -> dict[tuple[str, str], int]:
|
||||
"""(path, function) -> changed lines inside it, by the head's enclosing declarations."""
|
||||
counts: dict[tuple[str, str], int] = {}
|
||||
for path, ranges in changed.items():
|
||||
for lo, hi in ranges:
|
||||
for line in range(int(lo), int(hi) + 1):
|
||||
key = (path, index.function_at(path, line))
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def _body_calls(lines: Sequence[str] | None, language: str, function: str) -> set[str]:
|
||||
if not lines:
|
||||
return set()
|
||||
return {m.group(1) for i, line in enumerate(lines) if enclosing(lines, i, language) == function for m in _CALL.finditer(line)}
|
||||
|
||||
|
||||
def build_for_scan(
|
||||
root: Path,
|
||||
findings: Sequence[Any],
|
||||
engine_result: Mapping[str, Any] | None,
|
||||
*,
|
||||
base: Delta | None = None,
|
||||
rules: Sequence[PatternRule] | None = None,
|
||||
quick_languages: Iterable[str] | None = None,
|
||||
engine_languages: Iterable[str] | None = None,
|
||||
roles: RoleSet | None | bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""``static`` records for a checkout: one per (path, function, family) with a quick or engine signal. ``findings``
|
||||
are the quick findings (enabled and shadow alike), ``engine_result`` the engine's record (``findings`` with
|
||||
``site``/``family``/``rung``/``witness``, ``questions``, ``completed``, ``degradations``; ``None`` when the engine
|
||||
did not run, ``questions == 0`` is ``failed``). With ``base`` only functions enclosing a changed line are
|
||||
candidates, and ``delta.*`` compares their signals with the base's. ``roles``: ``True`` infers wrapper roles over
|
||||
the checkout (priors off), a :class:`.roles.RoleSet` is used as given, ``False``/``None`` leaves ``roles`` ``none``."""
|
||||
from ..plane.tasks.alerts import covers
|
||||
from ..plane.tasks.alerts import engine_languages as engine_covered
|
||||
from ..plane.tasks.alerts import quick_languages as quick_covered
|
||||
from ..semantic.facts import FactLoadError, load_facts
|
||||
|
||||
root = Path(root)
|
||||
loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR)
|
||||
by_id = {rule.rule_id: rule for rule in loaded}
|
||||
quick_langs = set(quick_languages if quick_languages is not None else quick_covered())
|
||||
engine_langs = set(engine_languages if engine_languages is not None else engine_covered())
|
||||
index = SourceIndex(root)
|
||||
quick, engine = _signals(index, findings, engine_result, by_id)
|
||||
keys = sorted(set(quick) | set(engine))
|
||||
changed: dict[tuple[str, str], int] = {}
|
||||
base_quick: dict[tuple[str, str, str], list[tuple[str, str]]] = {}
|
||||
base_engine: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {}
|
||||
base_index: SourceIndex | None = None
|
||||
if base is not None:
|
||||
changed = _changed_in(index, base.changed)
|
||||
keys = [k for k in keys if (k[0], k[1]) in changed]
|
||||
base_index = SourceIndex(base.root)
|
||||
base_quick, base_engine = _signals(base_index, base.findings, base.engine_result, by_id)
|
||||
entries, file_entries = entry_names(root, {k[0] for k in keys}) if keys else (set(), set())
|
||||
callers = index.callers() if keys else {}
|
||||
quick_version = ruleset_digest(loaded)
|
||||
engine_state = _engine_state(engine_result)
|
||||
completed, questions = (engine_result or {}).get("completed"), (engine_result or {}).get("questions")
|
||||
completion = int(completed) / int(questions) if isinstance(completed, int) and isinstance(questions, int) and questions else None
|
||||
degraded = bool((engine_result or {}).get("degradations"))
|
||||
vocabularies: dict[str, Vocabulary] = {}
|
||||
inferred = (infer_for_checkout(root) if keys else RoleSet(sources=("inferred_wrapper",))) if roles is True else roles or None
|
||||
try:
|
||||
sanitizer_calls = {language: {c for s in load_facts().for_language(language).sanitizers for c in s.calls} for language in LANGUAGES}
|
||||
except FactLoadError:
|
||||
sanitizer_calls = {}
|
||||
out: list[dict[str, Any]] = []
|
||||
for key in keys:
|
||||
path, function, family = key
|
||||
language = detect_language(Path(path))
|
||||
parts: dict[str, Part] = {}
|
||||
parts["quick"] = quick_part(quick.get(key, []), by_id, quick_version) if covers(quick_langs, language) else NONE
|
||||
if language in engine_langs and engine_state != "none":
|
||||
vocabulary = vocabularies.setdefault(language, Vocabulary.load(language))
|
||||
parts["engine"] = (
|
||||
engine_part(engine.get(key, []), completion=completion, degraded=degraded, vocabulary=vocabulary)
|
||||
if engine_state == "ran" else Part("failed")
|
||||
) # fmt: skip
|
||||
lines = index.lines(path)
|
||||
parts["source"] = source_part(lines, language, function)
|
||||
parts["roles"] = roles_part(function_of(lines, path, language, function), inferred, language)
|
||||
distance = 0 if function == GLOBAL and path in file_entries else entry_distance(function, entries, callers)
|
||||
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
|
||||
parts["facts"] = facts_part(_callers_of(index, callers, path, function), function)
|
||||
if base is None or base_index is None:
|
||||
parts["delta"] = NOT_APPLICABLE
|
||||
else:
|
||||
head_n = len(quick.get(key, [])) + len(engine.get(key, []))
|
||||
base_n = len(base_quick.get(key, [])) + len(base_engine.get(key, []))
|
||||
novelty = "new" if base_n == 0 else "worsened" if head_n > base_n else "unchanged"
|
||||
known = sanitizer_calls.get(language, set())
|
||||
removed = bool((_body_calls(base_index.lines(path), language, function) - _body_calls(lines, language, function)) & known)
|
||||
values = {"delta.novelty": novelty, "delta.changed_lines": changed.get((path, function), 0), "delta.sanitizer_removed": removed}
|
||||
parts["delta"] = Part("ran", values)
|
||||
unit, pin = ("delta", base.pin) if base is not None else ("pin", None)
|
||||
out.append(record(f"{path}::{function}", family, language, "static", parts, unit=unit, base=pin))
|
||||
return out
|
||||
|
||||
|
||||
def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]:
|
||||
"""The call sites of ``function`` in other product files, as repo-facts records them (capped at 40)."""
|
||||
if function == GLOBAL or function not in callers:
|
||||
@@ -567,8 +437,8 @@ def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str,
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Delta", "Part", "SourceIndex", "Vocabulary", "agree_part",
|
||||
"CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Part", "SourceIndex", "Vocabulary", "agree_part",
|
||||
"function_of", "model_sinks_part", "roles_part",
|
||||
"build_for_scan", "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "language_of", "quick_part", "record",
|
||||
"engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "quick_part", "record",
|
||||
"ruleset_digest", "source_part", "verify_part",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -38,7 +38,7 @@ class FoldError(RuntimeError):
|
||||
class Fold:
|
||||
"""One evaluation (or compile, or deployment) boundary: the groups evaluated, and what retrieval must not use."""
|
||||
|
||||
kind: str # outer | leave_one_source_out | leave_one_framework_out | compile | deployment
|
||||
kind: str # outer | compile | deployment
|
||||
name: str
|
||||
eval_groups: frozenset[str]
|
||||
held_source: str | None = None
|
||||
@@ -126,37 +126,6 @@ def outer_folds(examples: Sequence[Example], *, k: int = OUTER_K, seed: int = 0,
|
||||
return [Fold("outer", f"outer-{i}", frozenset(groups)) for i, groups in enumerate(assigned)]
|
||||
|
||||
|
||||
def leave_one_source_out(examples: Sequence[Example], *, exclude: Iterable[str] = ()) -> list[Fold]:
|
||||
excluded = set(exclude)
|
||||
by_source: dict[str, set[str]] = {}
|
||||
for example in examples:
|
||||
if example.group not in excluded:
|
||||
by_source.setdefault(example.source, set()).add(example.group)
|
||||
return [Fold("leave_one_source_out", f"source-{s}", frozenset(g), held_source=s) for s, g in sorted(by_source.items())]
|
||||
|
||||
|
||||
def leave_one_framework_out(
|
||||
examples: Sequence[Example], *, exclude: Iterable[str] = (), min_groups: int = FRAMEWORK_MIN_GROUPS
|
||||
) -> list[Fold]:
|
||||
"""One fold per framework seen; below ``min_groups`` groups it is ``insufficient data`` and evaluates nothing."""
|
||||
excluded = set(exclude)
|
||||
by_framework: dict[str, set[str]] = {}
|
||||
for example in examples:
|
||||
for framework in example.frameworks:
|
||||
if example.group not in excluded:
|
||||
by_framework.setdefault(framework, set()).add(example.group)
|
||||
folds = []
|
||||
for framework, groups in sorted(by_framework.items()):
|
||||
enough = len(groups) >= min_groups
|
||||
folds.append(
|
||||
Fold(
|
||||
"leave_one_framework_out", f"framework-{framework}", frozenset(groups) if enough else frozenset(),
|
||||
held_framework=framework, status="ok" if enough else INSUFFICIENT,
|
||||
)
|
||||
) # fmt: skip
|
||||
return folds
|
||||
|
||||
|
||||
def assert_disjoint(split: CompileSplit, folds: Iterable[Fold]) -> None:
|
||||
"""The compile split shares no group with any evaluation fold; outer folds are pairwise disjoint."""
|
||||
seen: dict[str, str] = {}
|
||||
@@ -181,6 +150,5 @@ def folds_digest(split: CompileSplit, folds: Iterable[Fold]) -> str:
|
||||
|
||||
__all__ = [
|
||||
"COMPILE_FRACTION", "FRAMEWORK_MIN_GROUPS", "INSUFFICIENT", "OUTER_K", "CompileSplit", "Fold", "FoldError", "GroupInfo",
|
||||
"assert_disjoint", "compile_split", "folds_digest", "group_table", "leave_one_framework_out", "leave_one_source_out",
|
||||
"outer_folds",
|
||||
"assert_disjoint", "compile_split", "folds_digest", "group_table", "outer_folds",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -23,7 +23,6 @@ finding. A role's ``path``/``function`` are provenance, never features.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
@@ -536,13 +535,9 @@ def _confidence(role: Role) -> float:
|
||||
return round(1.0 / (role.depth + 1), 6)
|
||||
|
||||
|
||||
def load_roles(path: Path) -> RoleSet:
|
||||
return RoleSet.from_json(json.loads(Path(path).read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CWE_OPERATION", "DEPENDENCY_DIRS", "MAX_DEPTH", "MODEL_ROLE_KINDS", "OPERATION_CWE", "ROLE_KINDS", "SOURCE_KIND", "VERSION",
|
||||
"Function", "Role", "RoleSet", "base_roles", "body_roles", "dependency_files", "from_model_roles", "functions_of",
|
||||
"infer_for_checkout", "infer_wrappers", "language_key", "load_roles", "mentions", "origin_of", "parameters",
|
||||
"infer_for_checkout", "infer_wrappers", "language_key", "mentions", "origin_of", "parameters",
|
||||
"vocabulary_overlay",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -35,31 +35,6 @@ class StaticHint:
|
||||
evidence_candidate: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MappingTaskRecord:
|
||||
kind: MappingKind
|
||||
path: str
|
||||
rule_id: str
|
||||
task: str
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DifferentialMappingRecord:
|
||||
path: str
|
||||
change_kind: str
|
||||
trust_boundary: str | None
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SharpEdgeRecord:
|
||||
path: str
|
||||
category: str
|
||||
surface: str
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntryPointRecord:
|
||||
path: str
|
||||
@@ -185,58 +160,6 @@ def attach_reachability_hints(targets: list[FileTarget], entry_points: list[Entr
|
||||
return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets]
|
||||
|
||||
|
||||
def verifier_evidence_candidates(hints: list[StaticHint], *, path: str | None = None) -> list[dict[str, object]]:
|
||||
selected = [hint for hint in hints if hint.evidence_candidate and (path is None or hint.path == path)]
|
||||
return [
|
||||
{
|
||||
"source": hint.analyzer,
|
||||
"rule_id": hint.rule_id,
|
||||
"path": hint.path,
|
||||
"line": hint.start_line,
|
||||
"severity": hint.severity,
|
||||
"message": hint.message,
|
||||
"provenance": hint.provenance,
|
||||
}
|
||||
for hint in selected
|
||||
]
|
||||
|
||||
|
||||
def semgrep_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
|
||||
return [
|
||||
MappingTaskRecord(
|
||||
kind="semgrep",
|
||||
path=hint.path,
|
||||
rule_id=hint.rule_id,
|
||||
task="pattern_variant_review",
|
||||
rationale="Use Semgrep evidence to check variants and rule precision before verifier promotion.",
|
||||
)
|
||||
for hint in hints
|
||||
if hint.analyzer == "semgrep"
|
||||
]
|
||||
|
||||
|
||||
def codeql_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
|
||||
return [
|
||||
MappingTaskRecord(
|
||||
kind="codeql",
|
||||
path=hint.path,
|
||||
rule_id=hint.rule_id,
|
||||
task="source_sink_sanitizer_path_review",
|
||||
rationale="Use CodeQL evidence to inspect source, sink, sanitizer, and path explanation.",
|
||||
)
|
||||
for hint in hints
|
||||
if hint.analyzer == "codeql"
|
||||
]
|
||||
|
||||
|
||||
def differential_mapping_record(path: str, *, change_kind: str, trust_boundary: str | None, rationale: str) -> DifferentialMappingRecord:
|
||||
return DifferentialMappingRecord(path=path, change_kind=change_kind, trust_boundary=trust_boundary, rationale=rationale)
|
||||
|
||||
|
||||
def sharp_edge_record(path: str, *, category: str, surface: str, rationale: str) -> SharpEdgeRecord:
|
||||
return SharpEdgeRecord(path=path, category=category, surface=surface, rationale=rationale)
|
||||
|
||||
|
||||
def write_static_hints(hints: list[StaticHint], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
@@ -87,12 +87,6 @@ def verdicts(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str
|
||||
return []
|
||||
|
||||
|
||||
def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str = "") -> Verdict | None:
|
||||
"""The strongest single verdict, kept for callers that ask one question and want one answer."""
|
||||
answers = verdicts(cpg, spec, function=function, file=file)
|
||||
return answers[0] if answers else None
|
||||
|
||||
|
||||
def _location(row: Mapping[str, object], file: str) -> str:
|
||||
"""``path:line:function`` of the setting, when the region named a file.
|
||||
|
||||
@@ -156,4 +150,4 @@ def _settings(rows: object, spec: ConfigSpec, *, function: str) -> list[dict[str
|
||||
return sorted(kept, key=lambda row: (str(row["method"]), _line(row), str(row["setting"])))
|
||||
|
||||
|
||||
__all__ = ["request_params", "verdict", "verdicts"]
|
||||
__all__ = ["request_params", "verdicts"]
|
||||
|
||||
@@ -29,12 +29,10 @@ from typing import Any
|
||||
|
||||
from ..cpg.backend import CpgResult
|
||||
from ..redaction import redact_secrets
|
||||
from .config_value import verdict as config_verdict
|
||||
from .config_value import verdicts as config_verdicts
|
||||
from .dominance import verdict as dominance_verdict
|
||||
from .ladder import Rung, Verdict
|
||||
from .specs import ConfigSpec, DominanceSpec, TaintSpec
|
||||
from .taint import verdict as taint_verdict
|
||||
from .taint import verdicts as taint_verdicts
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -93,34 +91,11 @@ def _arbitrate_all(
|
||||
if isinstance(spec, ConfigSpec):
|
||||
return config_verdicts(cpg, spec, function=function, file=path)
|
||||
if isinstance(spec, DominanceSpec):
|
||||
answer = _arbitrate(cpg, spec, path=path, function=function, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||
answer = dominance_verdict(cpg, spec, function=function, file=path)
|
||||
return [answer] if answer is not None else []
|
||||
return taint_verdicts(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||
|
||||
|
||||
def _arbitrate(
|
||||
cpg: CpgResult,
|
||||
spec: TaintSpec | DominanceSpec | ConfigSpec,
|
||||
*,
|
||||
path: str,
|
||||
function: str,
|
||||
parameter_sources: bool,
|
||||
call_depth: int,
|
||||
) -> Verdict | None:
|
||||
"""Route a family to the arbiter that can actually decide it.
|
||||
|
||||
Dispatching on the spec type rather than on a family name keeps this honest: a family without an arbiter
|
||||
cannot be silently routed to the wrong one. The pipeline first shipped taint-only, so access_control and
|
||||
config_secrets -- whose arbiters had worked since group 4 -- produced nothing, and every one of their
|
||||
pairs read as `both_silent` on the first end-to-end run.
|
||||
"""
|
||||
if isinstance(spec, DominanceSpec):
|
||||
return dominance_verdict(cpg, spec, function=function, file=path)
|
||||
if isinstance(spec, ConfigSpec):
|
||||
return config_verdict(cpg, spec, function=function, file=path)
|
||||
return taint_verdict(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||
|
||||
|
||||
def scan_region(
|
||||
cpg: CpgResult,
|
||||
spec: TaintSpec | DominanceSpec | ConfigSpec,
|
||||
|
||||
@@ -1,87 +1,9 @@
|
||||
"""Reporting that cannot flatter itself (model-grounded-detection, Req 9.3, 11.1).
|
||||
|
||||
Three disciplines survive from the deleted publish/scoring layer, and each is here because its absence caused
|
||||
a real error in this project's history rather than because it is tidy:
|
||||
|
||||
* **Per-slice rows, always.** A headline over mixed slices once hid that agent-written code scored 44.4%
|
||||
while the hand-written development slice scored 83.3%. `render` refuses to produce anything without rows.
|
||||
* **The overfitting gap beside the headline.** The distance between the development slice and the deciding
|
||||
real-world one is stated, never left for the reader to compute. When a slice is missing the gap is
|
||||
*absent*, not zero — reading a missing measurement as "no overfitting" is the flattering failure.
|
||||
* **The rung on every finding.** A finding's strength is what established it; omitting the rung invites a
|
||||
reader to treat a suspicion as a result.
|
||||
"""
|
||||
|
||||
"""Report model coverage for the scanned regions."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..findings import StaticFinding
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SliceRow:
|
||||
"""One slice's coverage. `covered` is pairs the model *split*, not pairs it flagged."""
|
||||
|
||||
slice: str
|
||||
pairs: int
|
||||
covered: int
|
||||
|
||||
@property
|
||||
def rate(self) -> float:
|
||||
return self.covered / self.pairs if self.pairs else 0.0
|
||||
|
||||
|
||||
def overfitting_gap(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> float | None:
|
||||
"""Development-slice coverage minus the deciding slice's, or ``None`` when either is absent.
|
||||
|
||||
``None`` rather than ``0.0`` on purpose: a slice that was never measured must not read as agreement
|
||||
between them.
|
||||
"""
|
||||
by_name = {row.slice: row for row in rows}
|
||||
if development not in by_name or deciding not in by_name:
|
||||
return None
|
||||
return by_name[development].rate - by_name[deciding].rate
|
||||
|
||||
|
||||
def render(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> str:
|
||||
"""The report. Raises when there are no per-slice rows, because a bare aggregate is the thing to prevent."""
|
||||
if not rows:
|
||||
raise ValueError("a report needs per-slice rows: there is no headline without them")
|
||||
|
||||
lines = ["## Model coverage", "", "| slice | pairs | covered | coverage |", "|---|---|---|---|"]
|
||||
for row in sorted(rows, key=lambda r: r.slice):
|
||||
lines.append(f"| {row.slice} | {row.pairs} | {row.covered} | {row.rate:.0%} |")
|
||||
|
||||
total_pairs = sum(row.pairs for row in rows)
|
||||
total_covered = sum(row.covered for row in rows)
|
||||
lines += [
|
||||
"",
|
||||
f"Across every slice: {total_covered}/{total_pairs} = {total_covered / total_pairs:.0%} — read the rows above it, not this line.",
|
||||
]
|
||||
|
||||
gap = overfitting_gap(rows, development=development, deciding=deciding)
|
||||
if gap is not None:
|
||||
lines += [
|
||||
"",
|
||||
f"**Overfitting gap: {gap:+.0%}** — `{development}` (development) minus `{deciding}`, which is the deciding real-world slice.",
|
||||
]
|
||||
elif development or deciding:
|
||||
lines += ["", f"**Overfitting gap: not measurable** — `{development}` or `{deciding}` is absent from this run."]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def render_findings(findings: Sequence[StaticFinding]) -> str:
|
||||
"""Findings with the rung that established each one (Req 5.3)."""
|
||||
lines = ["## Findings", "", "| finding | location | rung | severity |", "|---|---|---|---|"]
|
||||
for finding in findings:
|
||||
where = f"{finding.path}:{finding.line}" if finding.line else finding.path
|
||||
lines.append(f"| {finding.title} | {where} | `{finding.rung}` | {finding.severity} |")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
__all__ = ["SliceRow", "overfitting_gap", "render", "render_findings"]
|
||||
__all__ = []
|
||||
|
||||
|
||||
def coverage_rows(regions: Sequence[object]) -> list[dict[str, object]]:
|
||||
|
||||
@@ -5,7 +5,7 @@ graph establishes it and the LLM's claim is no longer load-bearing. A path a san
|
||||
`CORROBORATED`: the flow is real, but whether the sanitizer is *sufficient* is a semantic judgement the model
|
||||
cannot make, and that residual is the LLM's — checked against this verdict rather than averaged over runs.
|
||||
|
||||
No path at all yields `None`, not a negative verdict: the caller reports `suspicion`. And a query that failed
|
||||
No path at all yields an empty list, not a negative verdict: the caller reports `suspicion`. And a query that failed
|
||||
yields `None` too, because "the engine could not decide" is not "there is no flow" — conflating them is how a
|
||||
missing engine would silently become a clean bill of health.
|
||||
"""
|
||||
@@ -69,37 +69,6 @@ def request_params(
|
||||
}
|
||||
|
||||
|
||||
def verdict(
|
||||
cpg: CpgResult, spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False, call_depth: int = 0
|
||||
) -> Verdict | None:
|
||||
"""The strongest verdict the taint query supports for ``spec`` in ``function``, or ``None`` to stay at suspicion.
|
||||
|
||||
``parameter_sources`` additionally treats the labeled function's own parameters as untrusted. That is right
|
||||
for a function-level pair, where the function boundary *is* the trust boundary, and wrong for a whole
|
||||
repository, where most parameters carry internal values — so it is off by default and the caller opts in.
|
||||
An ENTRY POINT is the case where both are true at once: its parameters are attacker input by definition,
|
||||
which is what the driver uses to decide.
|
||||
|
||||
``call_depth`` admits sinks that many call-graph levels below the labeled method. Zero keeps the old
|
||||
function-local question; above zero is what lets a handler's parameter reach a sink in another module,
|
||||
which is the shape VAmPI's SQL injection has and the shape a function-scoped question cannot see.
|
||||
"""
|
||||
# The strongest of the per-site verdicts, which is what this always meant. It used to run its own
|
||||
# selection over the union of flows, and two copies of "which flow decides" is the duplication this
|
||||
# module has been bitten by before: the per-site preference added to `verdicts` did not reach here, so
|
||||
# the two entry points disagreed about the same rows.
|
||||
#
|
||||
# A flow the graph cannot see a discharge for is the finding; a sanitized or BOUNDED one is a question.
|
||||
# libpng bounds `strcpy(outname+len, ".png")` with `(len = strlen(inname)) > 250` against a char[256],
|
||||
# and entailing it is a claim the code contradicts. But the bound reaches the sink through an `error`
|
||||
# flag rather than by dominating it, so the graph has not shown the copy is guarded either -- and a
|
||||
# bound is no proof of safety in any case, off-by-one being the classic way they fail. Corroborated is
|
||||
# the honest rung: the model cannot assert this, so the judge is asked instead of the site being
|
||||
# silenced.
|
||||
answers = verdicts(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||
return answers[0] if answers else None
|
||||
|
||||
|
||||
def _usable_flows(
|
||||
cpg: CpgResult,
|
||||
spec: TaintSpec,
|
||||
@@ -137,17 +106,7 @@ def verdicts(
|
||||
parameter_sources: bool = False,
|
||||
call_depth: int = 0,
|
||||
) -> list[Verdict]:
|
||||
"""One verdict per distinct SINK SITE, strongest first.
|
||||
|
||||
``verdict`` answers "what is the strongest thing to say about this region", which is the right question
|
||||
for a pair: one labelled function, one bug, one answer. It is the wrong question for a region that spans
|
||||
a file. A PHP file with an SQL injection on line 6 and a command injection on line 13 got ONE injection
|
||||
verdict, and `system` won it on flow length -- so the SQL injection went unreported, by construction
|
||||
rather than by any failure of the analysis.
|
||||
|
||||
The first element is exactly what ``verdict`` returns, so the pair path and the committed measurements
|
||||
that rest on it are unchanged.
|
||||
"""
|
||||
"""One verdict per distinct sink site, strongest first."""
|
||||
flows = _usable_flows(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||
if not flows:
|
||||
return []
|
||||
@@ -164,7 +123,7 @@ def verdicts(
|
||||
preferred = [flow for flow in site_flows if _from_modelled_source(flow, spec)] or list(site_flows)
|
||||
open_flows = [flow for flow in preferred if not flow["sanitized"] and not flow["bounded"]]
|
||||
chosen.append(min(open_flows or preferred, key=lambda flow: _rank(flow, spec)))
|
||||
# Same total order the single-verdict path uses, so `verdicts(...)[0] == verdict(...)`.
|
||||
# Sort strongest first, then by site for stable results.
|
||||
chosen.sort(key=lambda flow: (bool(flow["sanitized"]) or bool(flow["bounded"]), _rank(flow, spec)))
|
||||
return [_verdict_for(flow, spec) for flow in chosen]
|
||||
|
||||
@@ -309,4 +268,4 @@ def _location(flow: Mapping[str, object]) -> str:
|
||||
return f"{where}:{line}:{flow.get('sinkMethod') or '?'}"
|
||||
|
||||
|
||||
__all__ = ["request_params", "verdict"]
|
||||
__all__ = ["request_params", "verdicts"]
|
||||
|
||||
@@ -13,7 +13,6 @@ from __future__ import annotations
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
FAMILY_IDS = (
|
||||
"injection",
|
||||
@@ -28,7 +27,6 @@ FAMILY_IDS = (
|
||||
"unknown",
|
||||
)
|
||||
VERIFIER_KINDS = ("canary", "static", "none")
|
||||
Relation = Literal["same", "parent_child", "lateral", "fabricated"]
|
||||
DEFAULT_FAMILIES_PATH = Path(__file__).resolve().parents[1] / "ruleset" / "families.toml"
|
||||
|
||||
_FAMILY_FIELDS = frozenset({"id", "description", "cwes", "mechanisms", "verifier", "parent", "limits"})
|
||||
@@ -72,24 +70,6 @@ class FamilyTaxonomy:
|
||||
def family_of_mechanism(self, mechanism: str) -> Family | None:
|
||||
return next((family for family in self.families if mechanism in family.mechanisms), None)
|
||||
|
||||
@property
|
||||
def hierarchical(self) -> bool:
|
||||
"""True when some family declares a parent. While this is False, partial credit can never be earned."""
|
||||
return any(family.parent for family in self.families)
|
||||
|
||||
def related(self, left: str, right: str) -> Relation:
|
||||
"""How two family answers relate. Abstention is not partial credit: ``unknown`` against a real
|
||||
family is ``lateral``, never ``parent_child``."""
|
||||
known = {family.id: family for family in self.families}
|
||||
if left not in known or right not in known:
|
||||
return "fabricated"
|
||||
if left == right:
|
||||
return "same"
|
||||
if known[left].parent == right or known[right].parent == left:
|
||||
return "parent_child"
|
||||
return "lateral"
|
||||
|
||||
|
||||
def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
|
||||
"""Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file);
|
||||
a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``."""
|
||||
@@ -189,7 +169,6 @@ __all__ = [
|
||||
"FamiliesError",
|
||||
"Family",
|
||||
"FamilyTaxonomy",
|
||||
"Relation",
|
||||
"load_families",
|
||||
]
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
|
||||
runs/<run>/<task>/output.tar the runner's PUT
|
||||
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
|
||||
runs/<run>/<task>/done.json status, bytes, file count, finished
|
||||
runs/<run>/inputs/<name> an input no task produces (``put_input``)
|
||||
|
||||
Nothing here logs or prints a URL.
|
||||
"""
|
||||
@@ -164,11 +163,6 @@ class Delivery:
|
||||
put.append(artifact)
|
||||
return put
|
||||
|
||||
def put_input(self, name: str, path: Path) -> str:
|
||||
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
|
||||
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
|
||||
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
|
||||
|
||||
# --- state ---------------------------------------------------------------------------------------------------
|
||||
|
||||
def save_state(self, data: Mapping[str, Any]) -> None:
|
||||
|
||||
@@ -1,61 +1,29 @@
|
||||
"""Per-task network egress on Agent Substrate (ai-service-plane, task 0: gateway + receiver Service).
|
||||
"""Per-task TLS egress to the store, Git hosts and model endpoints.
|
||||
|
||||
Every TCP connection an actor opens (DNS aside) is redirected to Substrate's egress gateway
|
||||
(``atenet-egress.ate-system``), which denies by default: an actor without an ``EgressPolicy`` gets no tunnel, and
|
||||
a refused connection is accepted and then closed. :func:`policy_for` grants a task exactly what it needs:
|
||||
|
||||
* ``http`` on port 80 to the artifact receiver's cluster name (``ousast-receiver.ax-system.svc.cluster.local``, a
|
||||
selector-less Service whose endpoint is this host's receiver, ``ops/ax/receiver-service.yaml.tmpl``). The actor
|
||||
cannot resolve cluster names, so the runner dials ``OUSAST_ARTIFACT_DIAL`` (the Service's ClusterIP, port 80)
|
||||
with the receiver's name as ``Host``. The gateway decides the policy on that ``Host`` but connects to the
|
||||
address the actor dialled (measured live: agentgateway's upstream is the CONNECT authority, not the Host), so
|
||||
the dial address must be one the gateway pod reaches: the ClusterIP, which kube-proxy maps to the host.
|
||||
* ``tls_passthrough`` on 443 for the Git hosts of the bound Workspaces' ``https`` repos (smart HTTP talks to the
|
||||
repo's host only) and for the bound Model's ``openultrasast.io/egress-hosts`` (``api.deepseek.com``).
|
||||
|
||||
Nothing else: no wildcard, no IP address (the gateway rejects them), no ``https`` rule (that one intercepts TLS and
|
||||
the actor would have to trust the gateway CA, which ax templates do not mount). :class:`Egress` writes the policy
|
||||
with ``kubectl ate create egress-policy`` after ``ax apply`` created the actor, reads it back, and before resume
|
||||
waits ``OUSAST_EGRESS_SETTLE`` s (11) because the gateway caches a missing policy as deny for 10 s. The kubectl-ate
|
||||
CLI has no delete verb: Substrate's store deletes the policy with its actor (``ON DELETE CASCADE``), so
|
||||
:meth:`Egress.delete` runs after ``ax delete task`` and confirms the policy is gone.
|
||||
The gateway denies by default. Egress writes and checks each actor's policy before resume,
|
||||
then waits for the cached denial to expire. Deleting the actor removes its policy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterable, Mapping
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlsplit
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .manifests import Model, Task, Workspace
|
||||
|
||||
__all__ = [
|
||||
"INPUTS_PATH",
|
||||
"RECEIVER_HOST",
|
||||
"RECEIVER_PORT",
|
||||
"Egress",
|
||||
"Receiver",
|
||||
"git_hosts",
|
||||
"receiver_cluster_ip",
|
||||
"policy_for",
|
||||
"receiver_address",
|
||||
"rules_of",
|
||||
]
|
||||
|
||||
RECEIVER_HOST = "ousast-receiver.ax-system.svc.cluster.local"
|
||||
INPUTS_PATH = "/inputs/" # GET <receiver>/inputs/<producer>/<artifact>: a consumer fetches a declared input
|
||||
RECEIVER_PORT = 18090 # the receiver Service's targetPort; ops/ax/up.sh renders the Service with the same number
|
||||
_NOT_FOUND = ("not found", "notfound")
|
||||
|
||||
|
||||
@@ -89,12 +57,10 @@ def policy_for(
|
||||
workspaces: Iterable[Workspace],
|
||||
model: Model | None,
|
||||
store_host: str | None,
|
||||
receiver_host: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
|
||||
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
|
||||
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless
|
||||
``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under
|
||||
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. An http Git repo cannot be reached under
|
||||
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
|
||||
hostnames only)."""
|
||||
tls, plain = git_hosts(workspaces)
|
||||
@@ -106,8 +72,6 @@ def policy_for(
|
||||
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
|
||||
hosts.add(store_host.lower())
|
||||
rules: list[dict[str, Any]] = []
|
||||
if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it)
|
||||
rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}})
|
||||
if hosts:
|
||||
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
|
||||
return {"rules": rules}
|
||||
@@ -125,41 +89,6 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||||
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
|
||||
|
||||
|
||||
def receiver_cluster_ip(context: str) -> str | None:
|
||||
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when
|
||||
kubectl cannot say."""
|
||||
if not context:
|
||||
raise RuntimeError("receiver_cluster_ip needs the profile's kube context")
|
||||
argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"]
|
||||
try:
|
||||
proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
address = proc.stdout.strip()
|
||||
return address if proc.returncode == 0 and _is_ip(address) else None
|
||||
|
||||
|
||||
def receiver_address(port: int, context: str | None) -> tuple[str, str | None]:
|
||||
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the
|
||||
profile's kube context the Service's ClusterIP is looked up in.
|
||||
|
||||
``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://<host>:<port>/`` and no dial (tests, a host the actor
|
||||
reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the
|
||||
Service's ClusterIP on port 80: the gateway checks the policy against ``Host`` but connects to the address the
|
||||
actor dialled (its CONNECT authority), so that address must reach the receiver from the gateway pod.
|
||||
"""
|
||||
host = os.environ.get("OUSAST_ARTIFACT_HOST")
|
||||
if host:
|
||||
return f"http://{host}:{port}/", None
|
||||
dial = os.environ.get("OUSAST_ARTIFACT_DIAL")
|
||||
if not dial:
|
||||
if not context:
|
||||
raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up")
|
||||
address = receiver_cluster_ip(context)
|
||||
dial = f"{address}:80" if address else None
|
||||
return f"http://{RECEIVER_HOST}/", dial
|
||||
|
||||
|
||||
class Egress:
|
||||
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
|
||||
|
||||
@@ -230,95 +159,3 @@ class Egress:
|
||||
time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11))
|
||||
|
||||
|
||||
class Receiver(ThreadingHTTPServer):
|
||||
"""Accepts one tar per running task and extracts it under ``<run dir>/<task>/`` (POST); serves a running task
|
||||
the producer artifacts it declared as inputs (``GET /inputs/<producer>/<artifact>``, ``inputs[task]``)."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None:
|
||||
super().__init__(("0.0.0.0", port), _Handler)
|
||||
self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]()
|
||||
self.inputs: dict[str, frozenset[str]] = {}
|
||||
self.lock = threading.Lock()
|
||||
self.url, self.dial = receiver_address(self.server_address[1], context)
|
||||
|
||||
def _task(self, headers: Any) -> str | None:
|
||||
"""The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None."""
|
||||
task = str(headers.get("X-Ousast-Task", ""))
|
||||
with self.lock:
|
||||
return task if headers.get("X-Ousast-Run") == self.run_name and task in self.running else None
|
||||
|
||||
def input_file(self, headers: Any, ref: str) -> tuple[int, str | Path]:
|
||||
"""(200, the file) when ``ref`` is one of the requesting task's declared inputs, else (403/404, why).
|
||||
|
||||
``ref`` must equal a declared ``<producer>/<artifact>`` exactly and resolve to a regular file under the run
|
||||
directory; nothing is normalised first, so ``..`` or an absolute path never names a declared input."""
|
||||
task = self._task(headers)
|
||||
if task is None:
|
||||
return 403, "unknown run or task"
|
||||
with self.lock:
|
||||
declared = ref in self.inputs.get(task, frozenset())
|
||||
if not declared:
|
||||
return 403, f"{ref!r} is not an input of {task}"
|
||||
root = self.base.resolve()
|
||||
path = (root / ref).resolve()
|
||||
if not path.is_relative_to(root) or not path.is_file():
|
||||
return 404, f"{ref} was not delivered"
|
||||
return 200, path
|
||||
|
||||
def accept(self, headers: Any, body: bytes) -> tuple[int, str]:
|
||||
task = self._task(headers)
|
||||
if task is None:
|
||||
return 403, "unknown run or task"
|
||||
if not str(headers.get("Content-Type", "")).startswith("application/x-tar"):
|
||||
return 415, "expected application/x-tar"
|
||||
dest = (self.base / task).resolve()
|
||||
try:
|
||||
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as tar:
|
||||
members = tar.getmembers()
|
||||
for m in members:
|
||||
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
|
||||
return 400, f"rejected member {m.name}"
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
tar.extractall(dest, members) # every member checked above
|
||||
except tarfile.TarError as exc:
|
||||
return 400, f"bad tar: {exc}"
|
||||
with self.lock:
|
||||
self.delivered.add(task)
|
||||
return 200, "ok"
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802 - http.server's naming
|
||||
server: Receiver = self.server # type: ignore[assignment]
|
||||
path = urlsplit(self.path).path
|
||||
if not path.startswith(INPUTS_PATH):
|
||||
self._text(404, "not found")
|
||||
return
|
||||
code, found = server.input_file(self.headers, unquote(path[len(INPUTS_PATH) :]))
|
||||
if not isinstance(found, Path):
|
||||
self._text(code, found)
|
||||
return
|
||||
with found.open("rb") as source: # streamed: a producer artifact can be far larger than any env value
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(os.fstat(source.fileno()).st_size))
|
||||
self.end_headers()
|
||||
shutil.copyfileobj(source, self.wfile, 1 << 16)
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802 - http.server's naming
|
||||
server: Receiver = self.server # type: ignore[assignment]
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
|
||||
self._text(*server.accept(self.headers, body))
|
||||
|
||||
def _text(self, code: int, text: str) -> None:
|
||||
data = text.encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
@@ -10,8 +10,7 @@ credential from this process's environment, never from a file (Req 4.6), and the
|
||||
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
|
||||
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
|
||||
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
|
||||
delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind
|
||||
profile for group 2 only (task 2.6 removes it).
|
||||
delivery fail it; ``ax delete task`` follows.
|
||||
|
||||
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
|
||||
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
|
||||
@@ -40,13 +39,12 @@ from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
|
||||
from .delivery import Delivery, DeliveryError, declared
|
||||
from .doctor import doctor
|
||||
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for
|
||||
from .egress import Egress, policy_for
|
||||
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
|
||||
from .memory import open_store
|
||||
from .profile import PlaneProfile, load_profile
|
||||
@@ -142,16 +140,14 @@ def _model_doc(model: Model) -> dict[str, Any]:
|
||||
return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
|
||||
|
||||
|
||||
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]:
|
||||
def render_task(run: Run, entry: RunTask, manifests: Manifests) -> list[dict[str, Any]]:
|
||||
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
|
||||
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
|
||||
(``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given."""
|
||||
(``OUSAST_INPUTS``) after the start by presigned URL."""
|
||||
task: Task = manifests.tasks[entry.task]
|
||||
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
|
||||
env = {e.name: e.value for e in task.env}
|
||||
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
|
||||
if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env
|
||||
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
|
||||
budget = _fields(entry.budget) if entry.budget else {}
|
||||
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
|
||||
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
|
||||
@@ -341,25 +337,22 @@ def _execute(
|
||||
base: Path,
|
||||
delivery: Delivery,
|
||||
router: Router,
|
||||
receiver: Receiver | None = None,
|
||||
) -> tuple[str, str]:
|
||||
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
|
||||
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None)
|
||||
docs = render_task(run, entry, manifests)
|
||||
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
|
||||
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
|
||||
workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
|
||||
body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
||||
store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None)
|
||||
if receiver is None:
|
||||
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
||||
body = delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
||||
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
||||
|
||||
def delivered() -> bool:
|
||||
return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name)
|
||||
return delivery.delivered(entry.name)
|
||||
|
||||
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
|
||||
policy = policy_for(task, workspaces, model, store_host, receiver_host)
|
||||
policy = policy_for(task, workspaces, model, delivery.host)
|
||||
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
|
||||
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
|
||||
egress.apply(ax_name, atespace, policy)
|
||||
@@ -369,7 +362,7 @@ def _execute(
|
||||
return "failed", str(exc)
|
||||
try:
|
||||
failure = _await(ax, ax_name, delivered, start)
|
||||
if failure is None and receiver is None:
|
||||
if failure is None:
|
||||
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
|
||||
except DeliveryError as exc:
|
||||
failure = str(exc)
|
||||
@@ -408,11 +401,6 @@ def run(
|
||||
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
|
||||
mirror=delivery.save_state,
|
||||
)
|
||||
receiver: Receiver | None = None
|
||||
if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it)
|
||||
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
|
||||
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
|
||||
threading.Thread(target=receiver.serve_forever, daemon=True).start()
|
||||
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
|
||||
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
|
||||
for entry in run_spec.tasks:
|
||||
@@ -427,7 +415,7 @@ def run(
|
||||
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
|
||||
state.set(entry.name, status="running", started=_now(), finished=None)
|
||||
try:
|
||||
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver)
|
||||
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router)
|
||||
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
|
||||
outcome = ("failed", f"{type(exc).__name__}: {exc}")
|
||||
finished.put((entry.name, *outcome))
|
||||
@@ -444,27 +432,17 @@ def run(
|
||||
pending.remove(entry)
|
||||
running[entry.name] = entry
|
||||
blocked.add(entry.serialize or "")
|
||||
if receiver is not None:
|
||||
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
|
||||
receiver.running.add(entry.name)
|
||||
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
|
||||
threading.Thread(target=worker, args=(entry,), daemon=True).start()
|
||||
if not running:
|
||||
break # nothing can start: a producer stopped short
|
||||
name, status, reason = finished.get()
|
||||
del running[name]
|
||||
if receiver is not None:
|
||||
with receiver.lock:
|
||||
receiver.running.discard(name)
|
||||
receiver.inputs.pop(name, None)
|
||||
summary = _read_json(base / name / "summary.json") or {}
|
||||
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
|
||||
halted = halted or status == "failed"
|
||||
if status == "unfinished":
|
||||
pending = [t for t in pending if name not in t.producers]
|
||||
finally:
|
||||
if receiver is not None:
|
||||
receiver.shutdown()
|
||||
lock.unlink(missing_ok=True)
|
||||
statuses = [state.status(t.name) for t in run_spec.tasks]
|
||||
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
|
||||
@@ -546,4 +524,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||
__all__ = ["Ax", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||
|
||||
@@ -25,8 +25,7 @@ The command runs as a child process, ``python -m openultrasast.plane.tasks.<spec
|
||||
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
|
||||
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
|
||||
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
|
||||
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` (or POSTs it to ``OUSAST_ARTIFACT_URL``, the receiver
|
||||
path kept until task 2.6 removes it) -- that delivery is the task's
|
||||
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` -- that delivery is the task's
|
||||
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
|
||||
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
|
||||
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
|
||||
@@ -48,13 +47,9 @@ Environment the runner reads, all optional except the two ax variables:
|
||||
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
|
||||
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
|
||||
OUSAST_GIT_PINS JSON ``{"<workspace>/<git name>": "<40-hex commit>"}`` from the reconciler
|
||||
OUSAST_ARTIFACT_URL, OUSAST_RUN where to deliver, and the run name carried in ``X-Ousast-Run``
|
||||
OUSAST_ARTIFACT_DIAL ``host:port`` to connect to instead of the URL's host, which is sent as
|
||||
``Host`` (the egress gateway decides on and resolves that name)
|
||||
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
|
||||
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
|
||||
command each is fetched from ``delivery.inputs[NAME]`` (else from
|
||||
``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` with the receiver's dial and headers)
|
||||
command each is fetched from ``delivery.inputs[NAME]``
|
||||
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
|
||||
fetch delivers a failed summary naming the input, the command never runs
|
||||
|
||||
@@ -700,16 +695,6 @@ def build_tar(output_dir: Path) -> bytes:
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[str, dict[str, str]]:
|
||||
"""(URL to connect to, headers): with ``dial`` the connection goes there and the URL's host is sent as ``Host``."""
|
||||
out = dict(headers)
|
||||
if not dial:
|
||||
return url, out
|
||||
parts = urlsplit(url)
|
||||
out["Host"] = parts.netloc
|
||||
return parts._replace(netloc=dial).geturl(), out
|
||||
|
||||
|
||||
def safe_url(url: str) -> str:
|
||||
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
|
||||
parts = urlsplit(url)
|
||||
@@ -723,18 +708,14 @@ def deliver(
|
||||
*,
|
||||
attempts: int = DELIVERY_ATTEMPTS,
|
||||
backoff: float = 1.0,
|
||||
dial: str | None = None,
|
||||
method: str = "POST",
|
||||
) -> int:
|
||||
"""Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry
|
||||
with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query.
|
||||
"""PUT the payload with retries and doubling backoff; return the 2xx status.
|
||||
|
||||
``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it
|
||||
cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself.
|
||||
Log lines and errors carry the URL without its query.
|
||||
"""
|
||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
||||
last = "no attempt made"
|
||||
for attempt in range(1, attempts + 1):
|
||||
request = urllib.request.Request(target, data=payload, method=method, headers=headers)
|
||||
request = urllib.request.Request(url, data=payload, method="PUT", headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
|
||||
status = int(response.status)
|
||||
@@ -752,16 +733,16 @@ def deliver(
|
||||
|
||||
|
||||
def fetch(
|
||||
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0, dial: str | None = None
|
||||
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0
|
||||
) -> int:
|
||||
"""GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`.
|
||||
|
||||
Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed."""
|
||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
||||
last = "no attempt made"
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = dest.with_name(dest.name + ".part")
|
||||
for attempt in range(1, attempts + 1):
|
||||
request = urllib.request.Request(target, method="GET", headers=headers)
|
||||
request = urllib.request.Request(url, method="GET", headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310
|
||||
shutil.copyfileobj(response, out, 1 << 16)
|
||||
@@ -784,7 +765,7 @@ def fetch(
|
||||
|
||||
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
|
||||
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
|
||||
presigned GET, no headers) when the start request carries one, else from the receiver.
|
||||
presigned GET, no headers) carried by the start request.
|
||||
|
||||
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
|
||||
raw = environ.get("OUSAST_INPUTS")
|
||||
@@ -796,21 +777,15 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
|
||||
raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None
|
||||
if not isinstance(inputs, dict):
|
||||
raise RunnerError("OUSAST_INPUTS must map input names to <producer>/<artifact>")
|
||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
||||
headers = {"X-Ousast-Run": environ.get("OUSAST_RUN", ""), "X-Ousast-Task": environ.get("OUSAST_TASK", "")}
|
||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||
fetched: list[Path] = []
|
||||
for name, ref in sorted(inputs.items()):
|
||||
dest = environ.get(f"OUSAST_INPUT_{name}")
|
||||
presigned = delivery.inputs.get(name) if delivery is not None else None
|
||||
if not dest or not isinstance(ref, str) or not (presigned or url):
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}")
|
||||
if not dest or not isinstance(ref, str) or not presigned:
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL or OUSAST_INPUT_{name}")
|
||||
try:
|
||||
if presigned:
|
||||
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
||||
else:
|
||||
source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
|
||||
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
||||
except DeliveryError as exc:
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
|
||||
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
|
||||
@@ -818,23 +793,14 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
|
||||
return fetched
|
||||
|
||||
|
||||
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
||||
"""PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver;
|
||||
def _deliver_output(output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
||||
"""PUT the output tar to ``delivery.put`` when the start request carried one;
|
||||
False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
|
||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||
if delivery is not None and delivery.put:
|
||||
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT")
|
||||
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff)
|
||||
return True
|
||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
||||
if not url:
|
||||
return False
|
||||
# the Run's task name (``OUSAST_TASK``) is what the receiver knows; the ax Task is named ``<run>-<task>``
|
||||
headers = {"Content-Type": "application/x-tar", "X-Ousast-Task": environ.get("OUSAST_TASK") or task.metadata.name}
|
||||
run = environ.get("OUSAST_RUN")
|
||||
if run:
|
||||
headers["X-Ousast-Run"] = run
|
||||
deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# --- entrypoint --------------------------------------------------------------------------------------------------
|
||||
@@ -849,7 +815,7 @@ class _Session:
|
||||
|
||||
@property
|
||||
def marker(self) -> Path:
|
||||
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the receiver has it."""
|
||||
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the store has it."""
|
||||
return self.state_dir / f"{self.task.metadata.name}.done.json"
|
||||
|
||||
@property
|
||||
@@ -893,7 +859,7 @@ def _finish(
|
||||
"""
|
||||
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
|
||||
try:
|
||||
marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery)
|
||||
marker["delivered"] = _deliver_output(session.output_dir, env, delivery)
|
||||
except DeliveryError as exc:
|
||||
log.error("artifact delivery failed: %s", exc)
|
||||
marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
|
||||
@@ -938,7 +904,7 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
|
||||
return 0
|
||||
try:
|
||||
_materialise_once(session, env)
|
||||
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only)
|
||||
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it
|
||||
except RunnerError as exc:
|
||||
log.error("runner failed: %s", exc)
|
||||
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
|
||||
|
||||
@@ -199,22 +199,6 @@ def _model(summary: object) -> str | None:
|
||||
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
||||
|
||||
|
||||
def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]:
|
||||
""":func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present)."""
|
||||
final = base / f"{case}-final" / "agreed.json"
|
||||
return case_features(
|
||||
facts=_json(base / f"{case}-facts" / "facts.json"),
|
||||
passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES},
|
||||
models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES},
|
||||
agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"),
|
||||
alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"),
|
||||
alerts_summary=_json(base / f"{case}-alerts" / "summary.json"),
|
||||
workspace=workspace,
|
||||
facts_summary=_json(base / f"{case}-facts" / "summary.json"),
|
||||
model_roles=_json(base / f"{case}-roles" / "roles.json"),
|
||||
)
|
||||
|
||||
|
||||
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||
return Path(env[name]) if env.get(name) else None
|
||||
|
||||
@@ -266,7 +250,7 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"]
|
||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", ]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
"""Offline-first provenance checks for declared push evaluation populations.
|
||||
|
||||
This validates inputs, never detector outcomes or capability admission. Source bytes come
|
||||
from pinned Git blobs, so a dirty cached worktree cannot silently change the experiment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import replace
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from openultrasast.repos import RepoUnavailable, load_repo_recipe, resolve
|
||||
|
||||
|
||||
class InputError(ValueError):
|
||||
"""An input identity or label cannot be trusted."""
|
||||
|
||||
|
||||
def _blob(root: Path, commit: str, path: str) -> bytes:
|
||||
pure = PurePosixPath(path)
|
||||
if pure.is_absolute() or ".." in pure.parts or not path or "\\" in path:
|
||||
raise InputError(f"unsafe source path: {path!r}")
|
||||
# Git object reads neither follow worktree symlinks nor execute project code.
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "cat-file", "blob", f"{commit}:{path}"],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
if not result.stdout:
|
||||
raise InputError(f"empty source bytes: {path}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _snapshot(item: dict[str, Any], directory: Path, cache_root: Path | None, fetch: bool) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"id": item["id"], "status": "invalid_input", "bytes_read": 0, "files": []}
|
||||
try:
|
||||
recipe = load_repo_recipe(directory / item["recipe"])
|
||||
commit = item.get("commit", recipe.commit)
|
||||
if not isinstance(commit, str) or not re.fullmatch("[0-9a-f]{40}", commit):
|
||||
raise InputError("commit must be a full immutable SHA")
|
||||
recipe = replace(recipe, commit=commit)
|
||||
root = resolve(recipe, fetch=fetch, cache_root=cache_root)
|
||||
result.update(commit=commit, repository=recipe.url, license=recipe.license)
|
||||
files = item["files"]
|
||||
paths = [entry["path"] for entry in files]
|
||||
if recipe.license_file not in paths or len(paths) < 2 or len(paths) != len(set(paths)):
|
||||
raise InputError("declare a unique license file and at least one source file")
|
||||
for entry in files:
|
||||
content = _blob(root, commit, entry["path"])
|
||||
result["bytes_read"] += len(content)
|
||||
edit = entry.get("edit")
|
||||
if edit:
|
||||
if not isinstance(edit, dict):
|
||||
raise InputError("authored edit must be an object")
|
||||
if not edit.get("provenance", "").startswith("authored"):
|
||||
raise InputError("transformation must identify authored provenance")
|
||||
before, after = edit["before"].encode(), edit["after"].encode()
|
||||
if not before or content.count(before) != 1 or before == after:
|
||||
raise InputError(f"authored edit must match exactly once: {entry['path']}")
|
||||
if hashlib.sha256(content).hexdigest() != entry.get("original_sha256"):
|
||||
raise InputError(f"original hash mismatch: {entry['path']}")
|
||||
content = content.replace(before, after, 1)
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
if len(content) != entry["size"] or digest != entry["sha256"] or not content:
|
||||
raise InputError(f"source size/hash mismatch: {entry['path']}")
|
||||
result["files"].append({"path": entry["path"], "size": len(content), "sha256": digest})
|
||||
result["status"] = "ready"
|
||||
except RepoUnavailable:
|
||||
result.update(
|
||||
status="missing_prerequisite",
|
||||
problem=f"{recipe.name}: pinned checkout {commit} unavailable; rerun this validator with --fetch to fetch it",
|
||||
)
|
||||
except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as exc:
|
||||
result["problem"] = str(exc)
|
||||
return result
|
||||
|
||||
|
||||
def validate_manifest(path: Path, *, cache_root: Path | None = None, fetch: bool = False) -> dict[str, Any]:
|
||||
"""Retain every case, marking unavailable inputs and unreviewed labels explicitly.
|
||||
|
||||
`ready` means input provenance is verified. It never means a finding was reproduced.
|
||||
Explicit --fetch is required even if the ordinary repository network env var is set.
|
||||
"""
|
||||
manifest_bytes = path.read_bytes()
|
||||
data = json.loads(manifest_bytes)
|
||||
if not isinstance(data, dict) or data.get("schema_version") != 1:
|
||||
raise InputError("unsupported push input schema version")
|
||||
for field in ("snapshots", "cases"):
|
||||
if not isinstance(data.get(field), list) or not data[field]:
|
||||
raise InputError(f"{field} must be a nonempty list")
|
||||
for item in data[field]:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("id"), str) or not item["id"]:
|
||||
raise InputError(f"{field} entry requires an identity")
|
||||
snapshots = [_snapshot(item, path.parent, cache_root, fetch) for item in data["snapshots"]]
|
||||
indexed = {item["id"]: item for item in snapshots}
|
||||
if len(indexed) != len(snapshots):
|
||||
raise InputError("duplicate snapshot identity")
|
||||
cases = []
|
||||
ids: set[str] = set()
|
||||
for item in data["cases"]:
|
||||
case = dict(item, status="invalid_input", admission="experimental")
|
||||
try:
|
||||
if item["id"] in ids:
|
||||
raise InputError("duplicate case identity")
|
||||
ids.add(item["id"])
|
||||
if item["label"] not in {"vulnerable", "fixed", "benign", "regression", "unsupported", "unreviewed"}:
|
||||
raise InputError("invalid case label")
|
||||
if item["role"] not in {"development", "transfer", "regression", "envelope", "reserved_holdout"}:
|
||||
raise InputError("invalid workload role")
|
||||
if not item["capability"] or not item["provenance"]:
|
||||
raise InputError("capability and provenance are required")
|
||||
if item["label"] == "unreviewed" and not item["prerequisites"]:
|
||||
raise InputError("unreviewed labels require an explicit prerequisite")
|
||||
inputs = [indexed[item[key]] for key in ("base", "tip")]
|
||||
problems = [value.get("problem", "") for value in inputs if value["status"] != "ready"]
|
||||
if any(value["status"] == "invalid_input" for value in inputs):
|
||||
case["problem"] = "; ".join(problems)
|
||||
elif problems or item["prerequisites"]:
|
||||
case.update(status="missing_prerequisite", problem="; ".join(problems + item["prerequisites"]))
|
||||
else:
|
||||
case["status"] = "ready"
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
case["problem"] = str(exc)
|
||||
cases.append(case)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
|
||||
"population": len(cases),
|
||||
"capability_prerequisites": data.get("capability_prerequisites", {}),
|
||||
"snapshots": snapshots,
|
||||
"cases": cases,
|
||||
"note": "Input validation only; every capability remains experimental pending independent evaluation.",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("manifest", type=Path)
|
||||
parser.add_argument("--fetch", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = validate_manifest(args.manifest, fetch=args.fetch)
|
||||
except (ValueError, OSError) as exc:
|
||||
print(json.dumps({"status": "invalid_input", "problem": str(exc)}))
|
||||
return 2
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0 if all(case["status"] == "ready" for case in result["cases"]) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
from collections.abc import Collection
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .preprocess import FileTarget
|
||||
|
||||
@@ -28,11 +28,6 @@ class RankingScore:
|
||||
static_boosts: list[str]
|
||||
|
||||
|
||||
class RankerClient(Protocol):
|
||||
def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def rank_targets(targets: list[FileTarget], model_rankings: object | None = None, model_id: str | None = None) -> list[RankingScore]:
|
||||
model_by_path = parse_ranker_response(model_rankings, model_id) if model_rankings is not None else {}
|
||||
rankings: list[RankingScore] = []
|
||||
@@ -43,32 +38,6 @@ def rank_targets(targets: list[FileTarget], model_rankings: object | None = None
|
||||
return sorted(rankings, key=lambda item: (-item.priority, item.path))
|
||||
|
||||
|
||||
def rank_targets_with_model(
|
||||
targets: list[FileTarget],
|
||||
*,
|
||||
client: RankerClient,
|
||||
model: str,
|
||||
chunk_size: int = 150,
|
||||
timeout_seconds: int = 60,
|
||||
) -> list[RankingScore]:
|
||||
if chunk_size < 1:
|
||||
raise RankingError("chunk_size must be at least 1")
|
||||
model_rankings: list[dict[str, object]] = []
|
||||
for start in range(0, len(targets), chunk_size):
|
||||
chunk = targets[start : start + chunk_size]
|
||||
response = client.complete_json(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": _ranker_system_prompt()},
|
||||
{"role": "user", "content": json.dumps([_target_prompt_payload(target) for target in chunk], sort_keys=True)},
|
||||
],
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
for ranking in parse_ranker_response(response, model).values():
|
||||
model_rankings.append(asdict(ranking))
|
||||
return rank_targets(targets, {"rankings": model_rankings}, model_id=model)
|
||||
|
||||
|
||||
def write_rankings(rankings: list[RankingScore], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"rankings": [asdict(ranking) for ranking in rankings]}
|
||||
@@ -205,25 +174,6 @@ def _clamp_score(value: int) -> int:
|
||||
return min(5, max(1, value))
|
||||
|
||||
|
||||
def _ranker_system_prompt() -> str:
|
||||
return (
|
||||
"Score each file for security review priority. Return JSON only with a rankings array. "
|
||||
"Each item must include path, surface, influence, reachability, rationale, and optional static_boosts. "
|
||||
"Scores must be integers from 1 to 5."
|
||||
)
|
||||
|
||||
|
||||
def _target_prompt_payload(target: FileTarget) -> dict[str, object]:
|
||||
return {
|
||||
"path": target.path,
|
||||
"language": target.language,
|
||||
"loc": target.loc,
|
||||
"tags": target.tags,
|
||||
"has_fuzz_entry_point": target.has_fuzz_entry_point,
|
||||
"reachability_hints": target.reachability_hints,
|
||||
}
|
||||
|
||||
|
||||
def _reachability_from_hints(hints: list[dict[str, object]]) -> int:
|
||||
score = 0
|
||||
for hint in hints:
|
||||
|
||||
@@ -7,20 +7,14 @@ call is conservatively charged its reservation on resume, never silently free.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from ..plane.budget import BudgetExhausted, SpendBudget
|
||||
from ..plane.memory import open_store
|
||||
from .board import Board
|
||||
from .budget import SearchBudget
|
||||
|
||||
@@ -421,79 +415,3 @@ class Coordinator:
|
||||
@staticmethod
|
||||
def _refs(refs: Any) -> bool:
|
||||
return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs)
|
||||
|
||||
|
||||
class BatchExecutor:
|
||||
"""Adapter to the existing AX/VM transfer and checkpoint protocol.
|
||||
|
||||
Pass AXLane/DockerLane instances and a trusted worker command. No default
|
||||
worker is invented here: reason/explore entrypoints belong to group 5.
|
||||
The coordinator owns retries, so batch performs exactly one attempt.
|
||||
"""
|
||||
|
||||
def __init__(self, *, image: str, command: tuple[str, ...], output: Path, lanes: dict[str, Any]) -> None:
|
||||
batch = import_module("benchmarks.ax.batch")
|
||||
batch.validate_image(image)
|
||||
if not command:
|
||||
raise ValueError("worker command required")
|
||||
self.image, self.command, self.output, self.lanes = image, command, output, lanes
|
||||
|
||||
def __call__(self, task: SearchTask) -> dict[str, Any]:
|
||||
import copy
|
||||
|
||||
batch = import_module("benchmarks.ax.batch")
|
||||
|
||||
if task.lane not in self.lanes:
|
||||
return {"status": "sandbox_failure", "cost_usd": 0}
|
||||
data = json.dumps(asdict(task), sort_keys=True, allow_nan=False).encode()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
identity = task.search_id + "-" + task.id
|
||||
item = batch.Item(
|
||||
identity,
|
||||
{"SPEC_URL": data},
|
||||
"result.dat",
|
||||
self.command,
|
||||
{"SEARCH_STEP": task.step, "SEARCH_ID": task.search_id, "SEARCH_TASK_ID": task.id},
|
||||
digest,
|
||||
)
|
||||
|
||||
def validate(raw: bytes, output: Path) -> None:
|
||||
if len(raw) > 262144:
|
||||
raise ValueError("search result size exceeded")
|
||||
record = json.loads(raw)
|
||||
if not isinstance(record, dict) or record.get("status") not in (
|
||||
"ok",
|
||||
"sandbox_failure",
|
||||
"instrument_failure",
|
||||
"oom",
|
||||
"execution_failure",
|
||||
):
|
||||
raise ValueError("invalid search result")
|
||||
(output / "result.json").write_text(json.dumps(record))
|
||||
|
||||
workload = batch.Workload(self.image, frozenset({"SPEC_URL", "RESULT_URL"}), task.limits["task_wall_seconds"], validate)
|
||||
lane = copy.copy(self.lanes[task.lane])
|
||||
if hasattr(lane, "workload"):
|
||||
lane.workload = workload
|
||||
out = self.output / task.search_id / task.id
|
||||
batch.dispatch([item], workload, out, {task.lane: lane}, lanes=(task.lane,), max_attempts=1)
|
||||
record_path = out / (hashlib.sha256(identity.encode()).hexdigest() + ".json")
|
||||
if not record_path.exists():
|
||||
return {"status": "execution_failure", "cost_usd": 0}
|
||||
record: dict[str, Any] = json.loads(record_path.read_text())
|
||||
return record
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Offline checkpoint inspection, also useful before resuming with an executor."""
|
||||
parser = argparse.ArgumentParser(description="Inspect a search board checkpoint (no model calls)")
|
||||
parser.add_argument("--memory", required=True)
|
||||
parser.add_argument("--search-id", required=True)
|
||||
parser.add_argument("--head", required=True)
|
||||
args = parser.parse_args()
|
||||
board = Board.resume(open_store(args.memory), args.search_id, args.head, coordinator="inspect")
|
||||
print(board.snapshot(), end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -22,7 +22,7 @@ import zipfile
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import suppress
|
||||
from dataclasses import asdict, dataclass, field, replace
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -709,20 +709,3 @@ def verify(
|
||||
isolation_mode=mode,
|
||||
chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("affected", type=Path)
|
||||
parser.add_argument("safe", type=Path)
|
||||
parser.add_argument("artefact", type=Path)
|
||||
parser.add_argument("family")
|
||||
args = parser.parse_args()
|
||||
result = verify(Side(args.affected), Side(args.safe), args.artefact, args.family)
|
||||
print(json.dumps(asdict(result), indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -2,11 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol, cast
|
||||
@@ -15,12 +13,11 @@ import yaml
|
||||
|
||||
from ..model.endpoint import Prices
|
||||
from ..plane.budget import BudgetExhausted, SpendBudget, cost_of
|
||||
from ..provider.openrouter import OpenRouterChatClient, parse_json_content
|
||||
from ..provider.openrouter import parse_json_content
|
||||
from . import _sandbox
|
||||
from .budget import SearchBudget
|
||||
from .coordinator import SearchTask
|
||||
from .demo import DEMO_SCHEMA, FAMILY_ORACLES, MAX_DEMO_BYTES, load_demo, validate_demo, validate_oracle
|
||||
from .executor import ObjectStoreExecutor
|
||||
from .task_storage import exception_reason
|
||||
|
||||
|
||||
@@ -525,44 +522,3 @@ class Worker:
|
||||
"cost_usd": meter.spent,
|
||||
"isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--task", required=True, type=Path)
|
||||
parser.add_argument("--repo", required=True, type=Path)
|
||||
parser.add_argument("--demo", required=True, type=Path)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--input-per-m", required=True, type=float)
|
||||
parser.add_argument("--output-per-m", required=True, type=float)
|
||||
parser.add_argument("--command-put-url")
|
||||
parser.add_argument("--result-get-url")
|
||||
args = parser.parse_args()
|
||||
if bool(args.command_put_url) != bool(args.result_get_url):
|
||||
parser.error("both executor mailbox URLs are required")
|
||||
executor = ObjectStoreExecutor(args.command_put_url, args.result_get_url) if args.command_put_url else None
|
||||
task = SearchTask(**json.loads(args.task.read_text()))
|
||||
if task.step == "explore" and executor is None:
|
||||
parser.error("explore requires a separate repository executor")
|
||||
client = OpenRouterChatClient(
|
||||
api_key=os.environ["OPENROUTER_API_KEY"],
|
||||
base_url=os.environ.get("OPENROUTER_BASE_URL", OpenRouterChatClient.base_url),
|
||||
max_attempts=1,
|
||||
)
|
||||
worker = Worker(
|
||||
client,
|
||||
repo=args.repo,
|
||||
demo=args.demo,
|
||||
model=args.model,
|
||||
prices=Prices(args.input_per_m, args.input_per_m, args.output_per_m),
|
||||
executor=executor,
|
||||
)
|
||||
try:
|
||||
print(json.dumps(worker(task)))
|
||||
finally:
|
||||
if executor is not None:
|
||||
executor.submit("stop", {}, timeout_seconds=10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -15,12 +15,8 @@ from .findings import StaticFinding
|
||||
from .hunter_tools import (
|
||||
PathEscapesRepo,
|
||||
clamp_repo_path,
|
||||
entry_points,
|
||||
find_refs,
|
||||
flows,
|
||||
grep_repo,
|
||||
obligations,
|
||||
read_definition,
|
||||
read_file,
|
||||
)
|
||||
from .provider.openrouter import OpenRouterChatClient, OpenRouterError
|
||||
@@ -31,7 +27,6 @@ DEFAULT_MAX_STEPS = 4
|
||||
CLIENT_ENV = "OPENULTRASAST_HUNTER_CLIENT"
|
||||
_FINDING_ID_PREFIX = "tool-hunter:"
|
||||
DEFAULT_TOOL_NAMES = ("read_file", "grep_repo", "find_refs")
|
||||
_HUNTER_TOOL_NAMES = frozenset(DEFAULT_TOOL_NAMES) | {"read_definition", "entry_points", "flows", "obligations"}
|
||||
_SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"})
|
||||
_SCRIPTED_FLAGS = frozenset({"scripted", "script"})
|
||||
_DUMP_FLAGS = frozenset({"dump", "dump-only"})
|
||||
@@ -91,50 +86,6 @@ HUNTER_TOOLS: list[dict[str, object]] = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_definition",
|
||||
"description": "Resolve one named function to its file, line span and body. Use it instead of guessing what a callee does.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"symbol": {"type": "string"}, "path": {"type": "string"}, "max_chars": {"type": "integer"}},
|
||||
"required": ["symbol"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "entry_points",
|
||||
"description": "List the handlers this repository exposes, with their access classification. Identifiers and line spans only.",
|
||||
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": []},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "flows",
|
||||
"description": "List adjudicated source-to-sink records inside one function. Identifiers and line numbers only.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}, "function": {"type": "string"}},
|
||||
"required": ["path", "function"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "obligations",
|
||||
"description": "List what one function owes and has not discharged: the operation, the missing discharger and why it is owed.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}, "function": {"type": "string"}},
|
||||
"required": ["path", "function"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
@@ -330,15 +281,13 @@ def run_tool_hunter(
|
||||
system_prompt: str = _SYSTEM_PROMPT,
|
||||
user_prompt: str | None = None,
|
||||
context_files: Sequence[str] = (),
|
||||
tools: Sequence[str] | None = None,
|
||||
max_chars: int = _DEFAULT_MAX_CHARS,
|
||||
tags: Sequence[str] = ("tool-hunter",),
|
||||
) -> list[StaticFinding]:
|
||||
"""One bounded hunt. Every keyword defaults to today's behaviour; a per-family detector overrides the
|
||||
prompt, the context it starts from, the tools it may call and the tags its findings carry."""
|
||||
"""One bounded hunt with a prompt, starting context and finding tags."""
|
||||
if max_steps <= 0:
|
||||
return []
|
||||
schemas = _tool_schemas(tools)
|
||||
schemas = HUNTER_TOOLS
|
||||
messages: list[dict[str, object]] = [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": _user_message(root, hotspots, user_prompt, context_files, max_chars)},
|
||||
@@ -350,7 +299,7 @@ def run_tool_hunter(
|
||||
if tool_calls:
|
||||
messages.append(_assistant_tool_message(response.content, tool_calls, response.reasoning))
|
||||
for call in tool_calls:
|
||||
if call.name in _HUNTER_TOOL_NAMES:
|
||||
if call.name in DEFAULT_TOOL_NAMES:
|
||||
used_tools = True
|
||||
messages.append(_tool_result_message(root, call, max_chars))
|
||||
continue
|
||||
@@ -414,12 +363,6 @@ def _looks_like_json(content: str) -> bool:
|
||||
return stripped.startswith(("[", "{"))
|
||||
|
||||
|
||||
def _tool_schemas(names: Sequence[str] | None) -> list[dict[str, object]]:
|
||||
wanted = tuple(names) if names is not None else DEFAULT_TOOL_NAMES
|
||||
by_name = {str(schema["function"]["name"]): schema for schema in HUNTER_TOOLS} # type: ignore[index]
|
||||
return [by_name[name] for name in wanted if name in by_name]
|
||||
|
||||
|
||||
def _user_message(root: Path, hotspots: Sequence[Hotspot], user_prompt: str | None, context_files: Sequence[str], max_chars: int) -> str:
|
||||
# Context files FIRST, the question LAST. DeepSeek bills a cached prefix at 1/30 of a fresh one, and the cache
|
||||
# is by prefix: with the question first, every candidate in the same file broke the prefix at its first
|
||||
@@ -502,30 +445,13 @@ def _dispatch_tool(root: Path, name: str, arguments: Mapping[str, object], defau
|
||||
path = _require_str(arguments, "path")
|
||||
max_chars = _optional_int(arguments, "max_chars", default_max_chars)
|
||||
return {"path": path, "content": read_file(root, path, max_chars=max_chars)}
|
||||
if name == "read_definition":
|
||||
symbol = _require_str(arguments, "symbol")
|
||||
scope = arguments.get("path")
|
||||
found = read_definition(
|
||||
root,
|
||||
symbol,
|
||||
max_chars=_optional_int(arguments, "max_chars", default_max_chars),
|
||||
path=str(scope) if isinstance(scope, str) and scope else None,
|
||||
)
|
||||
return found if found is not None else {"symbol": symbol, "found": False}
|
||||
if name == "entry_points":
|
||||
scope = arguments.get("path")
|
||||
return entry_points(root, path=str(scope) if isinstance(scope, str) and scope else None)
|
||||
if name == "flows":
|
||||
return flows(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function"))
|
||||
if name == "obligations":
|
||||
return obligations(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function"))
|
||||
if name == "grep_repo":
|
||||
pattern = _require_str(arguments, "pattern")
|
||||
max_matches = _optional_int(arguments, "max_matches", _DEFAULT_MAX_MATCHES)
|
||||
return grep_repo(root, pattern, max_matches=max_matches)
|
||||
if name == "find_refs":
|
||||
symbol = _require_str(arguments, "symbol")
|
||||
return find_refs(root, symbol, None)
|
||||
return find_refs(root, symbol)
|
||||
raise ValueError(f"unknown tool: {name}")
|
||||
|
||||
|
||||
|
||||
@@ -36,16 +36,6 @@ class VerificationResult:
|
||||
context_sources: list[str]
|
||||
|
||||
|
||||
def validate_evidence_transition(current: EvidenceLevel | str, target: EvidenceLevel | str) -> EvidenceLevel:
|
||||
current_level = EvidenceLevel(current)
|
||||
target_level = EvidenceLevel(target)
|
||||
if _EVIDENCE_ORDER[target_level] < _EVIDENCE_ORDER[current_level]:
|
||||
raise ValueError(f"cannot move evidence level backward: {current_level} -> {target_level}")
|
||||
if _EVIDENCE_ORDER[target_level] - _EVIDENCE_ORDER[current_level] > 1:
|
||||
raise ValueError(f"cannot skip evidence levels: {current_level} -> {target_level}")
|
||||
return target_level
|
||||
|
||||
|
||||
def is_report_verified(evidence_level: EvidenceLevel | str, status: VerificationStatus | str) -> bool:
|
||||
level = EvidenceLevel(evidence_level)
|
||||
verification_status = VerificationStatus(status)
|
||||
|
||||
Reference in New Issue
Block a user