chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery

Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts).
Deleted: the four hunter tools no caller enables (read_definition,
entry_points, flows, obligations) with their schemas and dispatch, the dead
mapping branch of find_refs; 20 functions whose only caller was one test
(mapping task records, compute_ranking_metrics, build_quick_hunter_prompt,
validate_evidence_transition, should_fuse, harness processors,
rank_targets_with_model, model/report slice rendering, taxonomy
hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles,
features.language_of and build_for_scan, folds.leave_one_*,
features.recorded_case, search BatchExecutor and the three __main__ shims,
delivery.put_input); the receiver delivery mode of the plane. push_inputs.py
moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses
it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282
tests collect. breadth.toml records the removals.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-13 11:06:09 +02:00
co-authored by Claude Fable 5.1
parent f5a904b7f2
commit f63703a604
60 changed files with 212 additions and 2576 deletions
+8 -7
View File
@@ -8,6 +8,7 @@
# parked = no work lands; scaffolding (stubs, unused flags, dead task modules) is removed; spec stays as history # parked = no work lands; scaffolding (stubs, unused flags, dead task modules) is removed; spec stays as history
# retiring = the "closes" deletion is scheduled; the spec directory goes with it # retiring = the "closes" deletion is scheduled; the spec directory goes with it
# review = Pathfinder 2026-10-13 decides between parked and retiring # review = Pathfinder 2026-10-13 decides between parked and retiring
# removed = the "closes" deletion has run; the entry stays one release as the record of what went
# #
# open_fronts (the weekly number) = specs with status active or review # open_fronts (the weekly number) = specs with status active or review
# + capabilities with status review or retiring whose "closes" has not run. # + capabilities with status review or retiring whose "closes" has not run.
@@ -181,7 +182,7 @@ status = "active"
spec = "hunter-learning-loop" spec = "hunter-learning-loop"
modules = ["tool_hunter.py", "hunter_tools.py", "provider/openrouter.py", "model/endpoint.py", "plane/budget.py"] modules = ["tool_hunter.py", "hunter_tools.py", "provider/openrouter.py", "model/endpoint.py", "plane/budget.py"]
lines = 1700 lines = 1700
note = "Pathfinder U6 counts the chat-client classes; expected outcome: one client protocol, one metering path" note = "four never-enabled hunter tools removed 2026-10-13 (P0); P2 collapses the client stack to one client, one meter, one resolver"
[capability.learn] [capability.learn]
status = "active" status = "active"
@@ -198,7 +199,7 @@ lines = 4803
[capability.core] [capability.core]
status = "active" status = "active"
modules = ["cli.py", "config.py", "contracts.py", "preprocess.py", "findings.py", "rank.py", "mapping.py", "ruleset", "policy", "reports.py", "stages.py", "run.py", "provenance.py", "profiling.py"] modules = ["cli.py", "config.py", "contracts.py", "preprocess.py", "findings.py", "rank.py", "mapping.py", "ruleset", "policy", "reports.py", "stages.py", "run.py", "provenance.py", "profiling.py"]
note = "hunter.py is the regex quick-rule pool and should be renamed; harness.py processor machinery is unreachable (U1)" note = "hunter.py is the regex quick-rule pool and should be renamed"
[capability.pairs-and-benchmarks] [capability.pairs-and-benchmarks]
status = "active" status = "active"
@@ -214,13 +215,13 @@ lines = 4885
closes = "Pathfinder U8 lists which modules only the retired HarnessX loop used (engine_alerts, harvest, workspaces, profile are the suspects); delete those" closes = "Pathfinder U8 lists which modules only the retired HarnessX loop used (engine_alerts, harvest, workspaces, profile are the suspects); delete those"
[capability.regress] [capability.regress]
status = "review" status = "removed" # f5a904b, 2026-10-13: deep mode records regress_retired; proof is the search line
modules = ["regress"] modules = ["regress"]
lines = 549 lines = 549
closes = "Pathfinder U9: if search/ covers the triggerable verdict, delete regress/ and the deep-mode REGRESS branch in cli.py:839-880" closes = "Pathfinder U9: if search/ covers the triggerable verdict, delete regress/ and the deep-mode REGRESS branch in cli.py:839-880"
[capability.improve] [capability.improve]
status = "retiring" status = "removed" # 94e96c6, 2026-10-13
modules = ["improve"] modules = ["improve"]
lines = 1175 lines = 1175
note = "HarnessX-era improve lever; no active spec names it" note = "HarnessX-era improve lever; no active spec names it"
@@ -228,13 +229,13 @@ closes = "delete improve/, `ousast improve` (cli.py:237, 433, 1106), plane/tasks
[capability.scan-extras] [capability.scan-extras]
status = "review" status = "review"
modules = ["fusion.py", "verification.py", "calibration.py", "scoring", "harness.py (processors)"] modules = ["fusion.py", "verification.py", "calibration.py", "scoring"] # harness.py processor machinery removed 2026-10-13
closes = "U1 found fusion decisions never reach findings or score, verification runs before MAP, processors unreachable; decide keep-and-fix or delete per module" closes = "U1 found fusion decisions never reach findings or score, verification runs before MAP, processors unreachable; decide keep-and-fix or delete per module"
[capability.mcp-and-skills] [capability.mcp-and-skills]
status = "parked" status = "parked"
spec = "mcp-shift-left-agent" spec = "mcp-shift-left-agent"
modules = ["mcp.py", "skills.py", "index.py"] modules = ["mcp.py"] # skills.py and index.py removed f5a904b, 2026-10-13
lines = 830 lines = 830
closes = "see spec.mcp-shift-left-agent" closes = "see spec.mcp-shift-left-agent"
@@ -246,6 +247,6 @@ closes = "U10: keep gate.py (CI + plane loop); delete map_gate.py and pair_gate.
[capability.push-evaluation-instruments] [capability.push-evaluation-instruments]
status = "review" status = "review"
modules = ["push_inputs.py", "push_scoring.py"] modules = ["push_scoring.py"] # push_inputs.py moved to benchmarks/push 2026-10-13; push_scoring stays (eligibility.py:18)
lines = 462 lines = 462
closes = "U3: move under benchmarks/push or delete if only tests use them" closes = "U3: move under benchmarks/push or delete if only tests use them"
+1 -1
View File
@@ -1,7 +1,7 @@
# Reproducible push evaluation inputs # Reproducible push evaluation inputs
`inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run. `inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run.
Run `.venv/bin/python -m openultrasast.push_inputs benchmarks/push/inputs.json`. Run `.venv/bin/python -m benchmarks.push.push_inputs benchmarks/push/inputs.json`.
It is offline by default, even if the ordinary repository network environment variable is set. It is offline by default, even if the ordinary repository network environment variable is set.
`--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits. `--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits.
No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one
+1 -1
View File
@@ -26,11 +26,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
from measure import git, pin_snapshot # noqa: E402 from measure import git, pin_snapshot # noqa: E402
from benchmarks.push.push_inputs import validate_manifest # noqa: E402
from openultrasast.model.scan import ScanBudget # noqa: E402 from openultrasast.model.scan import ScanBudget # noqa: E402
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402 from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
from openultrasast.push.runner import _provenance # noqa: E402 from openultrasast.push.runner import _provenance # noqa: E402
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402 from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
from openultrasast.push_inputs import validate_manifest # noqa: E402
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402 from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
TARGET = {"path": "app/routes/contributions.js", "function": "handleContributionsUpdate", "family": "injection"} TARGET = {"path": "app/routes/contributions.js", "function": "handleContributionsUpdate", "family": "injection"}
+2 -2
View File
@@ -13,10 +13,10 @@ import time
from dataclasses import replace from dataclasses import replace
from pathlib import Path from pathlib import Path
from benchmarks.push.push_inputs import validate_manifest
from openultrasast.model.contracts import QuestionIdentity from openultrasast.model.contracts import QuestionIdentity
from openultrasast.model.scan import ScanBudget from openultrasast.model.scan import ScanBudget
from openultrasast.push.runner import _provenance from openultrasast.push.runner import _provenance
from openultrasast.push_inputs import validate_manifest
from openultrasast.push_scoring import freeze_profile, score from openultrasast.push_scoring import freeze_profile, score
from openultrasast.repos import checkout_path, load_repo_recipe from openultrasast.repos import checkout_path, load_repo_recipe
@@ -314,7 +314,7 @@ def main() -> int:
"reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.", "reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.",
"selection": args.selection, "selection": args.selection,
"analysis_deadline_seconds": args.deadline, "analysis_deadline_seconds": args.deadline,
"recorded_cases": len(run["records"]), "case_count": len(run["records"]),
"declared_population": len(manifest["cases"]), "declared_population": len(manifest["cases"]),
"core_unchanged": True, "core_unchanged": True,
"adapter_changes": [], "adapter_changes": [],
+1 -1
View File
@@ -31,11 +31,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
from measure import git, pin_snapshot # noqa: E402 from measure import git, pin_snapshot # noqa: E402
from benchmarks.push.push_inputs import validate_manifest # noqa: E402
from openultrasast.model.scan import ScanBudget # noqa: E402 from openultrasast.model.scan import ScanBudget # noqa: E402
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402 from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
from openultrasast.push.runner import _provenance # noqa: E402 from openultrasast.push.runner import _provenance # noqa: E402
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402 from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
from openultrasast.push_inputs import validate_manifest # noqa: E402
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402 from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
# What each declared comparison is expected to show, stated before anything runs. `introduce` # What each declared comparison is expected to show, stated before anything runs. `introduce`
+1 -5
View File
@@ -26,11 +26,7 @@ flowchart TD
op -- "otherwise" --> adv["ADVISORY"] op -- "otherwise" --> adv["ADVISORY"]
``` ```
These decisions are meant for changed code. `learn/features.py` builds feature records in two ways: The plane task uses `learn/features.py` to build candidate feature records from delivered artifacts.
- for a whole-repository scan;
- for a base..head delta (`build_for_scan` with `Delta`). This is the same delta `ousast pre-push` checks.
Integration into scan and pre-push is still ahead. Integration into scan and pre-push is still ahead.
## 1. Instruments produce signals, not verdicts ## 1. Instruments produce signals, not verdicts
-4
View File
@@ -8,10 +8,6 @@ REPO="$(cd "$(dirname "$0")/../.." && pwd)"
PY="${OUSAST_PYTHON:-$REPO/.venv/bin/python}" PY="${OUSAST_PYTHON:-$REPO/.venv/bin/python}"
RUNNER_IMAGE="$(PYTHONPATH="$REPO/src" "$PY" -m openultrasast.plane.profile --profile "${OUSAST_PLANE_PROFILE:-kind}" --print images.runner)" RUNNER_IMAGE="$(PYTHONPATH="$REPO/src" "$PY" -m openultrasast.plane.profile --profile "${OUSAST_PLANE_PROFILE:-kind}" --print images.runner)"
export RUNNER_IMAGE export RUNNER_IMAGE
# Actors reach the receiver by its Service name through the egress gateway (ops/ax/receiver-service.yaml.tmpl,
# applied by up.sh, endpoint this host at OUSAST_ARTIFACT_PORT): the runner dials the Service's ClusterIP on port
# 80 with that name as Host; the reconciler looks the ClusterIP up unless OUSAST_ARTIFACT_DIAL is set.
export OUSAST_ARTIFACT_PORT="${OUSAST_ARTIFACT_PORT:-18090}"
envsubst < "$(dirname "$0")/smoke-run.yaml.tmpl" > "$SRC/smoke-run.yaml" envsubst < "$(dirname "$0")/smoke-run.yaml.tmpl" > "$SRC/smoke-run.yaml"
ousast() { PYTHONPATH="$REPO/src" "$PY" -m openultrasast.cli "$@"; } # this checkout's code, whatever python runs it ousast() { PYTHONPATH="$REPO/src" "$PY" -m openultrasast.cli "$@"; } # this checkout's code, whatever python runs it
ousast plane run "$SRC/smoke-run.yaml" "$@" ousast plane run "$SRC/smoke-run.yaml" "$@"
-42
View File
@@ -47,16 +47,6 @@ class FalsePositiveLearning:
demotion: float = 0.5 demotion: float = 0.5
@dataclass(frozen=True)
class RankingMetrics:
tier: str
verified_findings: int
false_positives: int
time_to_verification_seconds: float
missed_fixture_vulnerabilities: int
false_positive_reasons: dict[str, int]
@dataclass(frozen=True) @dataclass(frozen=True)
class RankingCalibration: class RankingCalibration:
path: str path: str
@@ -105,38 +95,6 @@ def learning_from_finding_outcome(
) )
def compute_ranking_metrics(events: list[dict[str, object]], *, tier: str) -> RankingMetrics:
verified = 0
false_positives = 0
reason_counts: dict[str, int] = {}
verification_times: list[float] = []
missed = 0
for event in events:
if event.get("tier") != tier:
continue
outcome = event.get("outcome")
if outcome == FindingOutcome.VERIFIED:
verified += 1
if outcome in _FALSE_POSITIVE_OUTCOMES:
false_positives += 1
reason = str(event.get("reason", FalsePositiveReason.UNVERIFIED))
reason_counts[reason] = reason_counts.get(reason, 0) + 1
duration = event.get("time_to_verification_seconds")
if isinstance(duration, int | float) and not isinstance(duration, bool):
verification_times.append(float(duration))
missed_value = event.get("missed_fixture_vulnerabilities", 0)
if isinstance(missed_value, int) and not isinstance(missed_value, bool):
missed += missed_value
return RankingMetrics(
tier=tier,
verified_findings=verified,
false_positives=false_positives,
time_to_verification_seconds=round(sum(verification_times) / len(verification_times), 2) if verification_times else 0.0,
missed_fixture_vulnerabilities=missed,
false_positive_reasons=dict(sorted(reason_counts.items())),
)
def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearning]) -> RankingCalibration: def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearning]) -> RankingCalibration:
applicable = [learning for learning in learnings if _learning_applies(ranking.path, learning)] applicable = [learning for learning in learnings if _learning_applies(ranking.path, learning)]
calibrated = ranking.priority calibrated = ranking.priority
+1 -1
View File
@@ -548,7 +548,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
config=config, config=config,
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets), trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets),
) )
write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json") write_harness_config(config=config, path=run.root / "harness.json")
runtime.start(mode=mode, target=run.target) runtime.start(mode=mode, target=run.target)
policy = runtime.run_stage("policy_load", load_policy) policy = runtime.run_stage("policy_load", load_policy)
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json" ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
-11
View File
@@ -99,17 +99,6 @@ def uncovered_languages(
return sorted(counts.items()) return sorted(counts.items())
def build_quick_hunter_prompt(target: FileTarget, source_excerpt: str) -> str:
return (
"Review this file for security-relevant issues. Return structured findings only when evidence exists.\n"
f"Path: {target.path}\n"
f"Language: {target.language}\n"
f"Tags: {', '.join(target.tags) or 'none'}\n"
"Source excerpt:\n"
f"{source_excerpt[:4000]}"
)
def write_findings(findings: list[StaticFinding], path: Path) -> None: def write_findings(findings: list[StaticFinding], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
payload = {"findings": [asdict(finding) for finding in findings]} payload = {"findings": [asdict(finding) for finding in findings]}
-5
View File
@@ -91,10 +91,6 @@ def fusion_triggers(
return sorted(triggers) return sorted(triggers)
def should_fuse(finding: StaticFinding, verification: VerificationResult | None = None, *, high_assurance: bool = False) -> bool:
return bool(fusion_triggers(finding, verification, high_assurance=high_assurance))
def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict: def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict:
"""Vulnerability-leaning panel: steel-mans the vulnerability case.""" """Vulnerability-leaning panel: steel-mans the vulnerability case."""
reachable = finding.reachability_status == "reachable" reachable = finding.reachability_status == "reachable"
@@ -213,5 +209,4 @@ __all__ = [
"fuse_finding", "fuse_finding",
"fuse_findings", "fuse_findings",
"fusion_triggers", "fusion_triggers",
"should_fuse",
] ]
+1 -77
View File
@@ -6,7 +6,7 @@ from collections.abc import Callable
from dataclasses import asdict, dataclass, field from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any, Literal, Protocol from typing import Any, Literal
from .config import ResolvedConfig from .config import ResolvedConfig
@@ -21,11 +21,6 @@ EventType = Literal[
"task_end", "task_end",
] ]
ContractMode = Literal["strict", "warn"]
class HarnessContractError(ValueError):
"""Raised when a processor violates its declared state contract."""
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -37,27 +32,9 @@ class HarnessEvent:
payload: dict[str, Any] = field(default_factory=dict) payload: dict[str, Any] = field(default_factory=dict)
@dataclass(frozen=True)
class ProcessorSpec:
name: str
version: str
handles: tuple[EventType, ...]
reads: tuple[str, ...] = ()
writes: tuple[str, ...] = ()
class Processor(Protocol):
spec: ProcessorSpec
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@dataclass(frozen=True) @dataclass(frozen=True)
class SerializedHarnessConfig: class SerializedHarnessConfig:
contract_mode: ContractMode
model_roles: dict[str, str | None] model_roles: dict[str, str | None]
processor_versions: dict[str, str]
prompt_hashes: dict[str, str] prompt_hashes: dict[str, str]
@@ -84,14 +61,10 @@ class HarnessRuntime:
scan_id: str, scan_id: str,
config: ResolvedConfig, config: ResolvedConfig,
trace_writer: HarnessTraceWriter, trace_writer: HarnessTraceWriter,
processors: list[Processor] | None = None,
contract_mode: ContractMode = "strict",
) -> None: ) -> None:
self.scan_id = scan_id self.scan_id = scan_id
self.config = config self.config = config
self.trace_writer = trace_writer self.trace_writer = trace_writer
self.processors = processors or []
self.contract_mode = contract_mode
self.state: dict[str, Any] = {"degradations": []} self.state: dict[str, Any] = {"degradations": []}
def start(self, *, mode: str, target: Path) -> None: def start(self, *, mode: str, target: Path) -> None:
@@ -119,71 +92,22 @@ class HarnessRuntime:
payload=payload, payload=payload,
) )
self.trace_writer.append(event) self.trace_writer.append(event)
for processor in self.processors:
if event_type in processor.spec.handles:
self._run_processor(processor, event)
def _run_processor(self, processor: Processor, event: HarnessEvent) -> None:
before = set(self.state)
missing_reads = [key for key in processor.spec.reads if key not in self.state]
if missing_reads:
self._contract_violation(processor.spec.name, f"missing reads: {', '.join(missing_reads)}")
return
updates = processor.handle(event, dict(self.state)) or {}
if not isinstance(updates, dict):
self._contract_violation(processor.spec.name, "processor returned a non-dict update")
return
undeclared = sorted(set(updates) - set(processor.spec.writes))
if undeclared:
self._contract_violation(processor.spec.name, f"undeclared writes: {', '.join(undeclared)}")
return
self.state.update(updates)
changed = sorted(set(self.state) - before | set(updates))
self.trace_writer.append(
HarnessEvent(
event_type=event.event_type,
scan_id=self.scan_id,
stage=event.stage,
timestamp=datetime.now(UTC).isoformat(),
payload={"processor": processor.spec.name, "changed_state": changed},
)
)
def _contract_violation(self, processor_name: str, reason: str) -> None:
message = f"processor {processor_name} contract violation: {reason}"
if self.contract_mode == "strict":
raise HarnessContractError(message)
self.state.setdefault("degradations", []).append(message)
self.trace_writer.append(
HarnessEvent(
event_type="stage_end",
scan_id=self.scan_id,
stage="processor_contract",
timestamp=datetime.now(UTC).isoformat(),
payload={"status": "degraded", "reason": message},
)
)
def write_harness_config( def write_harness_config(
*, *,
config: ResolvedConfig, config: ResolvedConfig,
processors: list[Processor],
contract_mode: ContractMode,
path: Path, path: Path,
prompts: dict[str, str] | None = None, prompts: dict[str, str] | None = None,
) -> None: ) -> None:
prompts = prompts or {} prompts = prompts or {}
serialized = SerializedHarnessConfig( serialized = SerializedHarnessConfig(
contract_mode=contract_mode,
model_roles={ model_roles={
"ranker": config.models.ranker, "ranker": config.models.ranker,
"hunter": config.models.hunter, "hunter": config.models.hunter,
"verifier": None, # schema kept byte for byte; LLM verification runs on the plane "verifier": None, # schema kept byte for byte; LLM verification runs on the plane
"patcher": None, # schema kept; `[models] patcher` was retired 2026-10-02 (never read) "patcher": None, # schema kept; `[models] patcher` was retired 2026-10-02 (never read)
}, },
processor_versions={processor.spec.name: processor.spec.version for processor in processors},
prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())}, prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())},
) )
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
+3 -237
View File
@@ -2,10 +2,10 @@ from __future__ import annotations
import os import os
import re import re
from collections.abc import Iterable, Iterator, Mapping from collections.abc import Iterator
from pathlib import Path from pathlib import Path
from .preprocess import FileTarget, enumerate_source_files from .preprocess import enumerate_source_files
class PathEscapesRepo(ValueError): class PathEscapesRepo(ValueError):
@@ -80,25 +80,9 @@ def grep_repo(root: Path, pattern: str, *, max_matches: int) -> list[dict[str, o
return matches return matches
def find_refs(root: Path, symbol: str, mapping_index: object) -> list[dict[str, object]]: def find_refs(root: Path, symbol: str) -> list[dict[str, object]]:
resolved_root = root.resolve() resolved_root = root.resolve()
refs: list[dict[str, object]] = [] refs: list[dict[str, object]] = []
for record in _iter_mapping_records(mapping_index):
if not _record_matches_symbol(record, symbol):
continue
raw_path = _record_path(record)
if raw_path is None:
continue
clamped = clamp_repo_path(resolved_root, raw_path)
name = _record_value(record, "name", "function_name", "symbol")
refs.append(
{
"path": clamped.relative_to(resolved_root).as_posix(),
"name": str(name) if name is not None else symbol,
"line": _as_line(_record_value(record, "line")),
"source": "mapping",
}
)
if symbol: if symbol:
compiled = re.compile(rf"\b{re.escape(symbol)}\b") compiled = re.compile(rf"\b{re.escape(symbol)}\b")
for relative, lines in _source_lines(resolved_root): for relative, lines in _source_lines(resolved_root):
@@ -113,221 +97,3 @@ def _iter_clamped_source_files(root: Path) -> Iterator[tuple[str, Path]]:
for path in enumerate_source_files(resolved_root): for path in enumerate_source_files(resolved_root):
relative = path.relative_to(resolved_root).as_posix() relative = path.relative_to(resolved_root).as_posix()
yield relative, clamp_repo_path(resolved_root, relative) yield relative, clamp_repo_path(resolved_root, relative)
def _iter_mapping_records(mapping_index: object) -> Iterable[object]:
if mapping_index is None:
return ()
if isinstance(mapping_index, Mapping):
return (mapping_index,)
if isinstance(mapping_index, (str, bytes, bytearray)):
return ()
if isinstance(mapping_index, Iterable):
return mapping_index
return (mapping_index,)
def _record_matches_symbol(record: object, symbol: str) -> bool:
for field in ("name", "function_name", "symbol"):
value = _record_value(record, field)
if value is not None and str(value) == symbol:
return True
return False
def _record_path(record: object) -> str | None:
value = record.get("path") if isinstance(record, Mapping) else getattr(record, "path", None)
if value is None:
return None
return str(value)
def _record_value(record: object, *names: str) -> object | None:
for name in names:
raw: object | None
if isinstance(record, Mapping):
if name not in record:
continue
raw = record[name]
else:
raw = getattr(record, name, None)
if raw is not None:
return raw
return None
def _as_line(value: object) -> int | None:
if isinstance(value, bool) or value is None:
return None
if isinstance(value, int):
return value
return None
# --- curated structural tools (learning-harness, Req 6.4, 6.5) --------------------------------------
# A detector reads code with `read_file` and `read_definition`; the three list tools below answer with
# identifiers and line spans only, so they can never become a second way to dump the repository. Every
# path a caller supplies is clamped to the scan root first. No caching: a detector calls these a handful
# of times per region, and a stale view of a tree that a round may have rewritten is worse than a reparse.
def read_definition(root: Path, symbol: str, *, max_chars: int, path: str | None = None) -> dict[str, object] | None:
"""The definition of one named function: where it is, how far it runs, and its text. None when absent."""
resolved_root = root.resolve()
scope = clamp_repo_path(resolved_root, path) if path is not None else None
for relative, target in _iter_clamped_source_files(resolved_root):
if scope is not None and target != scope:
continue
text = target.read_text(errors="ignore")
span = _named_span(relative, text, symbol)
if span is None:
continue
lines = text.splitlines()
start, end = span
while start > 1 and lines[start - 2].lstrip().startswith("@"):
start -= 1 # a handler's decorators are part of what the detector must see
body = "\n".join(lines[start - 1 : end])
return {"path": relative, "name": symbol, "start_line": start, "end_line": end, "text": body[: max(max_chars, 0)]}
return None
def entry_points(root: Path, *, path: str | None = None) -> list[dict[str, object]]:
"""Handlers the mapper names, with their access classification and evidence. No source text."""
from .mapping import analyze_entry_points
resolved_root = root.resolve()
relative = _relative_or_none(resolved_root, path)
targets = _repo_targets(resolved_root)
rows: list[dict[str, object]] = []
for record in analyze_entry_points(resolved_root, targets):
if relative is not None and record.path != relative:
continue
rows.append(
{
"path": record.path,
"function": record.function_name,
"name": record.name,
"kind": record.kind,
"line": record.line,
"end_line": record.end_line,
"access_level": record.access_level,
"evidence": list(record.access_evidence),
}
)
return rows
def flows(root: Path, *, path: str, function: str) -> list[dict[str, object]]:
"""Adjudicated source-to-sink records inside one function: which source, which sink, what the overlay said."""
from .findings import quick_scan_findings
from .rank import rank_targets
from .semantic import adjudicate
resolved_root = root.resolve()
relative = _relative_or_none(resolved_root, path)
targets = _repo_targets(resolved_root)
span = _span_in(resolved_root, relative, function)
if span is None:
return []
findings = [item for item in quick_scan_findings(resolved_root, targets, rank_targets(targets), None) if item.status != "shadow"]
rows: list[dict[str, object]] = []
for record in adjudicate(root=resolved_root, targets=targets, findings=findings):
if record.path != relative or record.line is None or not span[0] <= record.line <= span[1]:
continue
rows.append(
{
"path": record.path,
"function": function,
"line": record.line,
"sources": list(record.sources),
"sinks": list(record.sinks),
"disposition": record.disposition,
"cwe": record.cwe,
}
)
return rows
def obligations(root: Path, *, path: str, function: str) -> list[dict[str, object]]:
"""What one function owes and has not discharged: the operation, the missing discharger, why it is owed."""
from .config import ObligationsConfig
from .semantic.facts import FactLoadError, load_facts
from .semantic.ir import parse_file
from .semantic.obligations import check_obligations, load_obligation_facts
from .semantic.obligations.dominance import OrderDominance
resolved_root = root.resolve()
relative = _relative_or_none(resolved_root, path)
targets = _repo_targets(resolved_root)
try:
flow_facts = load_facts()
except FactLoadError:
return []
irs: dict[str, tuple[object, str]] = {}
texts: dict[str, str] = {}
for target in targets:
try:
text = (resolved_root / target.path).read_text(errors="ignore")
except OSError:
continue
texts[target.path] = text
irs[target.path] = (parse_file(target.path, text, target.language), text)
from .mapping import analyze_entry_points
result = check_obligations(
irs=irs, # type: ignore[arg-type]
entries=analyze_entry_points(resolved_root, targets),
facts=load_obligation_facts(),
flow_facts=flow_facts,
policy=None,
paths=(),
dominance=OrderDominance(texts=texts),
store_shapes=(),
min_siblings=ObligationsConfig().min_siblings,
)
return [
{
"path": finding.operation.path,
"function": finding.operation.function,
"line": finding.operation.line,
"operation": finding.operation.kind,
"resource": finding.operation.resource,
"missing": finding.missing,
"label": finding.label,
"evidence": list(finding.evidence),
}
for finding in result.findings
if (relative is None or finding.operation.path == relative) and finding.operation.function == function
]
def _repo_targets(root: Path) -> list[FileTarget]:
from .preprocess import preprocess_repository
_, targets = preprocess_repository(root)
return list(targets)
def _relative_or_none(root: Path, path: str | None) -> str | None:
if path is None:
return None
return clamp_repo_path(root, path).relative_to(root).as_posix()
def _named_span(relative: str, text: str, symbol: str) -> tuple[int, int] | None:
from .preprocess import detect_language
from .semantic.functions import named_function_ranges, spans_named
ranges = named_function_ranges(relative, text, detect_language(Path(relative)))
spans = spans_named(ranges or (), symbol)
return spans[0] if spans else None
def _span_in(root: Path, relative: str | None, function: str) -> tuple[int, int] | None:
if relative is None:
return None
try:
text = (root / relative).read_text(errors="ignore")
except OSError:
return None
return _named_span(relative, text, function)
+5 -135
View File
@@ -1,10 +1,6 @@
"""Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1). """Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1).
No model. The same per-instrument builders serve two callers: The plane task ``features`` (:mod:`..plane.tasks.features`) reads a case's delivered artifacts.
- the plane task ``features`` (:mod:`..plane.tasks.features`), which reads a case's delivered artifacts;
- the host: :func:`build_for_scan` over a checkout's quick findings and an engine result, for ``scan`` and
``pre-push`` (a whole-repository scan, or a base..head delta with :class:`Delta`).
A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument
emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the
@@ -26,8 +22,8 @@ from typing import Any
from ..model.taxonomy import load_families from ..model.taxonomy import load_families
from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files
from ..preprocess import build_file_target, detect_language from ..preprocess import build_file_target, detect_language
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset from ..ruleset import PatternRule
from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, infer_for_checkout, origin_of from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, origin_of
from .schema import ( from .schema import (
BY_NAME, BY_NAME,
LANGUAGES, LANGUAGES,
@@ -62,12 +58,6 @@ NONE = Part("none")
NOT_APPLICABLE = Part("not_applicable") NOT_APPLICABLE = Part("not_applicable")
def language_of(path: str) -> str:
"""The closed language of a path (its extension), ``other`` outside :data:`.schema.LANGUAGES`."""
language = detect_language(Path(path))
return language if language in LANGUAGES else "other"
def _clip(name: str, value: Any) -> Any: def _clip(name: str, value: Any) -> Any:
spec = BY_NAME[name] spec = BY_NAME[name]
if value is None or spec.type != "int" or isinstance(value, bool): if value is None or spec.type != "int" or isinstance(value, bool):
@@ -328,18 +318,6 @@ def function_of(lines: Sequence[str] | None, path: str, language: str, function:
# --- the host path --------------------------------------------------------------------------------------------------- # --- the host path ---------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class Delta:
"""A base..head delta: the base checkout, the head-side changed line ranges per path (``git diff -U0``), and the
base's quick findings and engine result, whose signals decide novelty."""
root: Path
changed: Mapping[str, Sequence[tuple[int, int]]]
findings: Sequence[Any] = ()
engine_result: Mapping[str, Any] | None = None
pin: str | None = None
class SourceIndex: class SourceIndex:
"""A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level """A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level
caller graph (the repo-facts patterns) for the entry-point distance.""" caller graph (the repo-facts patterns) for the entry-point distance."""
@@ -423,12 +401,6 @@ def _family_of(rule: PatternRule | None) -> str:
return family.id if family is not None else UNKNOWN_FAMILY return family.id if family is not None else UNKNOWN_FAMILY
def _engine_state(result: Mapping[str, Any] | None) -> str:
if result is None:
return "none"
return "ran" if int(result.get("questions") or 0) > 0 else "failed"
def _signals( def _signals(
index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule] index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule]
) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]: ) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]:
@@ -447,108 +419,6 @@ def _signals(
return quick, engine return quick, engine
def _changed_in(index: SourceIndex, changed: Mapping[str, Sequence[tuple[int, int]]]) -> dict[tuple[str, str], int]:
"""(path, function) -> changed lines inside it, by the head's enclosing declarations."""
counts: dict[tuple[str, str], int] = {}
for path, ranges in changed.items():
for lo, hi in ranges:
for line in range(int(lo), int(hi) + 1):
key = (path, index.function_at(path, line))
counts[key] = counts.get(key, 0) + 1
return counts
def _body_calls(lines: Sequence[str] | None, language: str, function: str) -> set[str]:
if not lines:
return set()
return {m.group(1) for i, line in enumerate(lines) if enclosing(lines, i, language) == function for m in _CALL.finditer(line)}
def build_for_scan(
root: Path,
findings: Sequence[Any],
engine_result: Mapping[str, Any] | None,
*,
base: Delta | None = None,
rules: Sequence[PatternRule] | None = None,
quick_languages: Iterable[str] | None = None,
engine_languages: Iterable[str] | None = None,
roles: RoleSet | None | bool = True,
) -> list[dict[str, Any]]:
"""``static`` records for a checkout: one per (path, function, family) with a quick or engine signal. ``findings``
are the quick findings (enabled and shadow alike), ``engine_result`` the engine's record (``findings`` with
``site``/``family``/``rung``/``witness``, ``questions``, ``completed``, ``degradations``; ``None`` when the engine
did not run, ``questions == 0`` is ``failed``). With ``base`` only functions enclosing a changed line are
candidates, and ``delta.*`` compares their signals with the base's. ``roles``: ``True`` infers wrapper roles over
the checkout (priors off), a :class:`.roles.RoleSet` is used as given, ``False``/``None`` leaves ``roles`` ``none``."""
from ..plane.tasks.alerts import covers
from ..plane.tasks.alerts import engine_languages as engine_covered
from ..plane.tasks.alerts import quick_languages as quick_covered
from ..semantic.facts import FactLoadError, load_facts
root = Path(root)
loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR)
by_id = {rule.rule_id: rule for rule in loaded}
quick_langs = set(quick_languages if quick_languages is not None else quick_covered())
engine_langs = set(engine_languages if engine_languages is not None else engine_covered())
index = SourceIndex(root)
quick, engine = _signals(index, findings, engine_result, by_id)
keys = sorted(set(quick) | set(engine))
changed: dict[tuple[str, str], int] = {}
base_quick: dict[tuple[str, str, str], list[tuple[str, str]]] = {}
base_engine: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {}
base_index: SourceIndex | None = None
if base is not None:
changed = _changed_in(index, base.changed)
keys = [k for k in keys if (k[0], k[1]) in changed]
base_index = SourceIndex(base.root)
base_quick, base_engine = _signals(base_index, base.findings, base.engine_result, by_id)
entries, file_entries = entry_names(root, {k[0] for k in keys}) if keys else (set(), set())
callers = index.callers() if keys else {}
quick_version = ruleset_digest(loaded)
engine_state = _engine_state(engine_result)
completed, questions = (engine_result or {}).get("completed"), (engine_result or {}).get("questions")
completion = int(completed) / int(questions) if isinstance(completed, int) and isinstance(questions, int) and questions else None
degraded = bool((engine_result or {}).get("degradations"))
vocabularies: dict[str, Vocabulary] = {}
inferred = (infer_for_checkout(root) if keys else RoleSet(sources=("inferred_wrapper",))) if roles is True else roles or None
try:
sanitizer_calls = {language: {c for s in load_facts().for_language(language).sanitizers for c in s.calls} for language in LANGUAGES}
except FactLoadError:
sanitizer_calls = {}
out: list[dict[str, Any]] = []
for key in keys:
path, function, family = key
language = detect_language(Path(path))
parts: dict[str, Part] = {}
parts["quick"] = quick_part(quick.get(key, []), by_id, quick_version) if covers(quick_langs, language) else NONE
if language in engine_langs and engine_state != "none":
vocabulary = vocabularies.setdefault(language, Vocabulary.load(language))
parts["engine"] = (
engine_part(engine.get(key, []), completion=completion, degraded=degraded, vocabulary=vocabulary)
if engine_state == "ran" else Part("failed")
) # fmt: skip
lines = index.lines(path)
parts["source"] = source_part(lines, language, function)
parts["roles"] = roles_part(function_of(lines, path, language, function), inferred, language)
distance = 0 if function == GLOBAL and path in file_entries else entry_distance(function, entries, callers)
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
parts["facts"] = facts_part(_callers_of(index, callers, path, function), function)
if base is None or base_index is None:
parts["delta"] = NOT_APPLICABLE
else:
head_n = len(quick.get(key, [])) + len(engine.get(key, []))
base_n = len(base_quick.get(key, [])) + len(base_engine.get(key, []))
novelty = "new" if base_n == 0 else "worsened" if head_n > base_n else "unchanged"
known = sanitizer_calls.get(language, set())
removed = bool((_body_calls(base_index.lines(path), language, function) - _body_calls(lines, language, function)) & known)
values = {"delta.novelty": novelty, "delta.changed_lines": changed.get((path, function), 0), "delta.sanitizer_removed": removed}
parts["delta"] = Part("ran", values)
unit, pin = ("delta", base.pin) if base is not None else ("pin", None)
out.append(record(f"{path}::{function}", family, language, "static", parts, unit=unit, base=pin))
return out
def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]: def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]:
"""The call sites of ``function`` in other product files, as repo-facts records them (capped at 40).""" """The call sites of ``function`` in other product files, as repo-facts records them (capped at 40)."""
if function == GLOBAL or function not in callers: if function == GLOBAL or function not in callers:
@@ -567,8 +437,8 @@ def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str,
__all__ = [ __all__ = [
"CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Delta", "Part", "SourceIndex", "Vocabulary", "agree_part", "CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Part", "SourceIndex", "Vocabulary", "agree_part",
"function_of", "model_sinks_part", "roles_part", "function_of", "model_sinks_part", "roles_part",
"build_for_scan", "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "language_of", "quick_part", "record", "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "quick_part", "record",
"ruleset_digest", "source_part", "verify_part", "ruleset_digest", "source_part", "verify_part",
] # fmt: skip ] # fmt: skip
+2 -34
View File
@@ -38,7 +38,7 @@ class FoldError(RuntimeError):
class Fold: class Fold:
"""One evaluation (or compile, or deployment) boundary: the groups evaluated, and what retrieval must not use.""" """One evaluation (or compile, or deployment) boundary: the groups evaluated, and what retrieval must not use."""
kind: str # outer | leave_one_source_out | leave_one_framework_out | compile | deployment kind: str # outer | compile | deployment
name: str name: str
eval_groups: frozenset[str] eval_groups: frozenset[str]
held_source: str | None = None held_source: str | None = None
@@ -126,37 +126,6 @@ def outer_folds(examples: Sequence[Example], *, k: int = OUTER_K, seed: int = 0,
return [Fold("outer", f"outer-{i}", frozenset(groups)) for i, groups in enumerate(assigned)] return [Fold("outer", f"outer-{i}", frozenset(groups)) for i, groups in enumerate(assigned)]
def leave_one_source_out(examples: Sequence[Example], *, exclude: Iterable[str] = ()) -> list[Fold]:
excluded = set(exclude)
by_source: dict[str, set[str]] = {}
for example in examples:
if example.group not in excluded:
by_source.setdefault(example.source, set()).add(example.group)
return [Fold("leave_one_source_out", f"source-{s}", frozenset(g), held_source=s) for s, g in sorted(by_source.items())]
def leave_one_framework_out(
examples: Sequence[Example], *, exclude: Iterable[str] = (), min_groups: int = FRAMEWORK_MIN_GROUPS
) -> list[Fold]:
"""One fold per framework seen; below ``min_groups`` groups it is ``insufficient data`` and evaluates nothing."""
excluded = set(exclude)
by_framework: dict[str, set[str]] = {}
for example in examples:
for framework in example.frameworks:
if example.group not in excluded:
by_framework.setdefault(framework, set()).add(example.group)
folds = []
for framework, groups in sorted(by_framework.items()):
enough = len(groups) >= min_groups
folds.append(
Fold(
"leave_one_framework_out", f"framework-{framework}", frozenset(groups) if enough else frozenset(),
held_framework=framework, status="ok" if enough else INSUFFICIENT,
)
) # fmt: skip
return folds
def assert_disjoint(split: CompileSplit, folds: Iterable[Fold]) -> None: def assert_disjoint(split: CompileSplit, folds: Iterable[Fold]) -> None:
"""The compile split shares no group with any evaluation fold; outer folds are pairwise disjoint.""" """The compile split shares no group with any evaluation fold; outer folds are pairwise disjoint."""
seen: dict[str, str] = {} seen: dict[str, str] = {}
@@ -181,6 +150,5 @@ def folds_digest(split: CompileSplit, folds: Iterable[Fold]) -> str:
__all__ = [ __all__ = [
"COMPILE_FRACTION", "FRAMEWORK_MIN_GROUPS", "INSUFFICIENT", "OUTER_K", "CompileSplit", "Fold", "FoldError", "GroupInfo", "COMPILE_FRACTION", "FRAMEWORK_MIN_GROUPS", "INSUFFICIENT", "OUTER_K", "CompileSplit", "Fold", "FoldError", "GroupInfo",
"assert_disjoint", "compile_split", "folds_digest", "group_table", "leave_one_framework_out", "leave_one_source_out", "assert_disjoint", "compile_split", "folds_digest", "group_table", "outer_folds",
"outer_folds",
] # fmt: skip ] # fmt: skip
+1 -6
View File
@@ -23,7 +23,6 @@ finding. A role's ``path``/``function`` are provenance, never features.
from __future__ import annotations from __future__ import annotations
import json
import re import re
from collections import Counter from collections import Counter
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
@@ -536,13 +535,9 @@ def _confidence(role: Role) -> float:
return round(1.0 / (role.depth + 1), 6) return round(1.0 / (role.depth + 1), 6)
def load_roles(path: Path) -> RoleSet:
return RoleSet.from_json(json.loads(Path(path).read_text(encoding="utf-8")))
__all__ = [ __all__ = [
"CWE_OPERATION", "DEPENDENCY_DIRS", "MAX_DEPTH", "MODEL_ROLE_KINDS", "OPERATION_CWE", "ROLE_KINDS", "SOURCE_KIND", "VERSION", "CWE_OPERATION", "DEPENDENCY_DIRS", "MAX_DEPTH", "MODEL_ROLE_KINDS", "OPERATION_CWE", "ROLE_KINDS", "SOURCE_KIND", "VERSION",
"Function", "Role", "RoleSet", "base_roles", "body_roles", "dependency_files", "from_model_roles", "functions_of", "Function", "Role", "RoleSet", "base_roles", "body_roles", "dependency_files", "from_model_roles", "functions_of",
"infer_for_checkout", "infer_wrappers", "language_key", "load_roles", "mentions", "origin_of", "parameters", "infer_for_checkout", "infer_wrappers", "language_key", "mentions", "origin_of", "parameters",
"vocabulary_overlay", "vocabulary_overlay",
] # fmt: skip ] # fmt: skip
-77
View File
@@ -35,31 +35,6 @@ class StaticHint:
evidence_candidate: bool = True evidence_candidate: bool = True
@dataclass(frozen=True)
class MappingTaskRecord:
kind: MappingKind
path: str
rule_id: str
task: str
rationale: str
@dataclass(frozen=True)
class DifferentialMappingRecord:
path: str
change_kind: str
trust_boundary: str | None
rationale: str
@dataclass(frozen=True)
class SharpEdgeRecord:
path: str
category: str
surface: str
rationale: str
@dataclass(frozen=True) @dataclass(frozen=True)
class EntryPointRecord: class EntryPointRecord:
path: str path: str
@@ -185,58 +160,6 @@ def attach_reachability_hints(targets: list[FileTarget], entry_points: list[Entr
return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets] return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets]
def verifier_evidence_candidates(hints: list[StaticHint], *, path: str | None = None) -> list[dict[str, object]]:
selected = [hint for hint in hints if hint.evidence_candidate and (path is None or hint.path == path)]
return [
{
"source": hint.analyzer,
"rule_id": hint.rule_id,
"path": hint.path,
"line": hint.start_line,
"severity": hint.severity,
"message": hint.message,
"provenance": hint.provenance,
}
for hint in selected
]
def semgrep_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
return [
MappingTaskRecord(
kind="semgrep",
path=hint.path,
rule_id=hint.rule_id,
task="pattern_variant_review",
rationale="Use Semgrep evidence to check variants and rule precision before verifier promotion.",
)
for hint in hints
if hint.analyzer == "semgrep"
]
def codeql_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
return [
MappingTaskRecord(
kind="codeql",
path=hint.path,
rule_id=hint.rule_id,
task="source_sink_sanitizer_path_review",
rationale="Use CodeQL evidence to inspect source, sink, sanitizer, and path explanation.",
)
for hint in hints
if hint.analyzer == "codeql"
]
def differential_mapping_record(path: str, *, change_kind: str, trust_boundary: str | None, rationale: str) -> DifferentialMappingRecord:
return DifferentialMappingRecord(path=path, change_kind=change_kind, trust_boundary=trust_boundary, rationale=rationale)
def sharp_edge_record(path: str, *, category: str, surface: str, rationale: str) -> SharpEdgeRecord:
return SharpEdgeRecord(path=path, category=category, surface=surface, rationale=rationale)
def write_static_hints(hints: list[StaticHint], path: Path) -> None: def write_static_hints(hints: list[StaticHint], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n") path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n")
+1 -7
View File
@@ -87,12 +87,6 @@ def verdicts(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str
return [] return []
def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str = "") -> Verdict | None:
"""The strongest single verdict, kept for callers that ask one question and want one answer."""
answers = verdicts(cpg, spec, function=function, file=file)
return answers[0] if answers else None
def _location(row: Mapping[str, object], file: str) -> str: def _location(row: Mapping[str, object], file: str) -> str:
"""``path:line:function`` of the setting, when the region named a file. """``path:line:function`` of the setting, when the region named a file.
@@ -156,4 +150,4 @@ def _settings(rows: object, spec: ConfigSpec, *, function: str) -> list[dict[str
return sorted(kept, key=lambda row: (str(row["method"]), _line(row), str(row["setting"]))) return sorted(kept, key=lambda row: (str(row["method"]), _line(row), str(row["setting"])))
__all__ = ["request_params", "verdict", "verdicts"] __all__ = ["request_params", "verdicts"]
+1 -26
View File
@@ -29,12 +29,10 @@ from typing import Any
from ..cpg.backend import CpgResult from ..cpg.backend import CpgResult
from ..redaction import redact_secrets from ..redaction import redact_secrets
from .config_value import verdict as config_verdict
from .config_value import verdicts as config_verdicts from .config_value import verdicts as config_verdicts
from .dominance import verdict as dominance_verdict from .dominance import verdict as dominance_verdict
from .ladder import Rung, Verdict from .ladder import Rung, Verdict
from .specs import ConfigSpec, DominanceSpec, TaintSpec from .specs import ConfigSpec, DominanceSpec, TaintSpec
from .taint import verdict as taint_verdict
from .taint import verdicts as taint_verdicts from .taint import verdicts as taint_verdicts
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -93,34 +91,11 @@ def _arbitrate_all(
if isinstance(spec, ConfigSpec): if isinstance(spec, ConfigSpec):
return config_verdicts(cpg, spec, function=function, file=path) return config_verdicts(cpg, spec, function=function, file=path)
if isinstance(spec, DominanceSpec): if isinstance(spec, DominanceSpec):
answer = _arbitrate(cpg, spec, path=path, function=function, parameter_sources=parameter_sources, call_depth=call_depth) answer = dominance_verdict(cpg, spec, function=function, file=path)
return [answer] if answer is not None else [] return [answer] if answer is not None else []
return taint_verdicts(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth) return taint_verdicts(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
def _arbitrate(
cpg: CpgResult,
spec: TaintSpec | DominanceSpec | ConfigSpec,
*,
path: str,
function: str,
parameter_sources: bool,
call_depth: int,
) -> Verdict | None:
"""Route a family to the arbiter that can actually decide it.
Dispatching on the spec type rather than on a family name keeps this honest: a family without an arbiter
cannot be silently routed to the wrong one. The pipeline first shipped taint-only, so access_control and
config_secrets -- whose arbiters had worked since group 4 -- produced nothing, and every one of their
pairs read as `both_silent` on the first end-to-end run.
"""
if isinstance(spec, DominanceSpec):
return dominance_verdict(cpg, spec, function=function, file=path)
if isinstance(spec, ConfigSpec):
return config_verdict(cpg, spec, function=function, file=path)
return taint_verdict(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
def scan_region( def scan_region(
cpg: CpgResult, cpg: CpgResult,
spec: TaintSpec | DominanceSpec | ConfigSpec, spec: TaintSpec | DominanceSpec | ConfigSpec,
+2 -80
View File
@@ -1,87 +1,9 @@
"""Reporting that cannot flatter itself (model-grounded-detection, Req 9.3, 11.1). """Report model coverage for the scanned regions."""
Three disciplines survive from the deleted publish/scoring layer, and each is here because its absence caused
a real error in this project's history rather than because it is tidy:
* **Per-slice rows, always.** A headline over mixed slices once hid that agent-written code scored 44.4%
while the hand-written development slice scored 83.3%. `render` refuses to produce anything without rows.
* **The overfitting gap beside the headline.** The distance between the development slice and the deciding
real-world one is stated, never left for the reader to compute. When a slice is missing the gap is
*absent*, not zero — reading a missing measurement as "no overfitting" is the flattering failure.
* **The rung on every finding.** A finding's strength is what established it; omitting the rung invites a
reader to treat a suspicion as a result.
"""
from __future__ import annotations from __future__ import annotations
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import dataclass
from ..findings import StaticFinding __all__ = []
@dataclass(frozen=True)
class SliceRow:
"""One slice's coverage. `covered` is pairs the model *split*, not pairs it flagged."""
slice: str
pairs: int
covered: int
@property
def rate(self) -> float:
return self.covered / self.pairs if self.pairs else 0.0
def overfitting_gap(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> float | None:
"""Development-slice coverage minus the deciding slice's, or ``None`` when either is absent.
``None`` rather than ``0.0`` on purpose: a slice that was never measured must not read as agreement
between them.
"""
by_name = {row.slice: row for row in rows}
if development not in by_name or deciding not in by_name:
return None
return by_name[development].rate - by_name[deciding].rate
def render(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> str:
"""The report. Raises when there are no per-slice rows, because a bare aggregate is the thing to prevent."""
if not rows:
raise ValueError("a report needs per-slice rows: there is no headline without them")
lines = ["## Model coverage", "", "| slice | pairs | covered | coverage |", "|---|---|---|---|"]
for row in sorted(rows, key=lambda r: r.slice):
lines.append(f"| {row.slice} | {row.pairs} | {row.covered} | {row.rate:.0%} |")
total_pairs = sum(row.pairs for row in rows)
total_covered = sum(row.covered for row in rows)
lines += [
"",
f"Across every slice: {total_covered}/{total_pairs} = {total_covered / total_pairs:.0%} — read the rows above it, not this line.",
]
gap = overfitting_gap(rows, development=development, deciding=deciding)
if gap is not None:
lines += [
"",
f"**Overfitting gap: {gap:+.0%}** — `{development}` (development) minus `{deciding}`, which is the deciding real-world slice.",
]
elif development or deciding:
lines += ["", f"**Overfitting gap: not measurable** — `{development}` or `{deciding}` is absent from this run."]
return "\n".join(lines) + "\n"
def render_findings(findings: Sequence[StaticFinding]) -> str:
"""Findings with the rung that established each one (Req 5.3)."""
lines = ["## Findings", "", "| finding | location | rung | severity |", "|---|---|---|---|"]
for finding in findings:
where = f"{finding.path}:{finding.line}" if finding.line else finding.path
lines.append(f"| {finding.title} | {where} | `{finding.rung}` | {finding.severity} |")
return "\n".join(lines) + "\n"
__all__ = ["SliceRow", "overfitting_gap", "render", "render_findings"]
def coverage_rows(regions: Sequence[object]) -> list[dict[str, object]]: def coverage_rows(regions: Sequence[object]) -> list[dict[str, object]]:
+4 -45
View File
@@ -5,7 +5,7 @@ graph establishes it and the LLM's claim is no longer load-bearing. A path a san
`CORROBORATED`: the flow is real, but whether the sanitizer is *sufficient* is a semantic judgement the model `CORROBORATED`: the flow is real, but whether the sanitizer is *sufficient* is a semantic judgement the model
cannot make, and that residual is the LLM's — checked against this verdict rather than averaged over runs. cannot make, and that residual is the LLM's — checked against this verdict rather than averaged over runs.
No path at all yields `None`, not a negative verdict: the caller reports `suspicion`. And a query that failed No path at all yields an empty list, not a negative verdict: the caller reports `suspicion`. And a query that failed
yields `None` too, because "the engine could not decide" is not "there is no flow" — conflating them is how a yields `None` too, because "the engine could not decide" is not "there is no flow" — conflating them is how a
missing engine would silently become a clean bill of health. missing engine would silently become a clean bill of health.
""" """
@@ -69,37 +69,6 @@ def request_params(
} }
def verdict(
cpg: CpgResult, spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False, call_depth: int = 0
) -> Verdict | None:
"""The strongest verdict the taint query supports for ``spec`` in ``function``, or ``None`` to stay at suspicion.
``parameter_sources`` additionally treats the labeled function's own parameters as untrusted. That is right
for a function-level pair, where the function boundary *is* the trust boundary, and wrong for a whole
repository, where most parameters carry internal values — so it is off by default and the caller opts in.
An ENTRY POINT is the case where both are true at once: its parameters are attacker input by definition,
which is what the driver uses to decide.
``call_depth`` admits sinks that many call-graph levels below the labeled method. Zero keeps the old
function-local question; above zero is what lets a handler's parameter reach a sink in another module,
which is the shape VAmPI's SQL injection has and the shape a function-scoped question cannot see.
"""
# The strongest of the per-site verdicts, which is what this always meant. It used to run its own
# selection over the union of flows, and two copies of "which flow decides" is the duplication this
# module has been bitten by before: the per-site preference added to `verdicts` did not reach here, so
# the two entry points disagreed about the same rows.
#
# A flow the graph cannot see a discharge for is the finding; a sanitized or BOUNDED one is a question.
# libpng bounds `strcpy(outname+len, ".png")` with `(len = strlen(inname)) > 250` against a char[256],
# and entailing it is a claim the code contradicts. But the bound reaches the sink through an `error`
# flag rather than by dominating it, so the graph has not shown the copy is guarded either -- and a
# bound is no proof of safety in any case, off-by-one being the classic way they fail. Corroborated is
# the honest rung: the model cannot assert this, so the judge is asked instead of the site being
# silenced.
answers = verdicts(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
return answers[0] if answers else None
def _usable_flows( def _usable_flows(
cpg: CpgResult, cpg: CpgResult,
spec: TaintSpec, spec: TaintSpec,
@@ -137,17 +106,7 @@ def verdicts(
parameter_sources: bool = False, parameter_sources: bool = False,
call_depth: int = 0, call_depth: int = 0,
) -> list[Verdict]: ) -> list[Verdict]:
"""One verdict per distinct SINK SITE, strongest first. """One verdict per distinct sink site, strongest first."""
``verdict`` answers "what is the strongest thing to say about this region", which is the right question
for a pair: one labelled function, one bug, one answer. It is the wrong question for a region that spans
a file. A PHP file with an SQL injection on line 6 and a command injection on line 13 got ONE injection
verdict, and `system` won it on flow length -- so the SQL injection went unreported, by construction
rather than by any failure of the analysis.
The first element is exactly what ``verdict`` returns, so the pair path and the committed measurements
that rest on it are unchanged.
"""
flows = _usable_flows(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth) flows = _usable_flows(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
if not flows: if not flows:
return [] return []
@@ -164,7 +123,7 @@ def verdicts(
preferred = [flow for flow in site_flows if _from_modelled_source(flow, spec)] or list(site_flows) preferred = [flow for flow in site_flows if _from_modelled_source(flow, spec)] or list(site_flows)
open_flows = [flow for flow in preferred if not flow["sanitized"] and not flow["bounded"]] open_flows = [flow for flow in preferred if not flow["sanitized"] and not flow["bounded"]]
chosen.append(min(open_flows or preferred, key=lambda flow: _rank(flow, spec))) chosen.append(min(open_flows or preferred, key=lambda flow: _rank(flow, spec)))
# Same total order the single-verdict path uses, so `verdicts(...)[0] == verdict(...)`. # Sort strongest first, then by site for stable results.
chosen.sort(key=lambda flow: (bool(flow["sanitized"]) or bool(flow["bounded"]), _rank(flow, spec))) chosen.sort(key=lambda flow: (bool(flow["sanitized"]) or bool(flow["bounded"]), _rank(flow, spec)))
return [_verdict_for(flow, spec) for flow in chosen] return [_verdict_for(flow, spec) for flow in chosen]
@@ -309,4 +268,4 @@ def _location(flow: Mapping[str, object]) -> str:
return f"{where}:{line}:{flow.get('sinkMethod') or '?'}" return f"{where}:{line}:{flow.get('sinkMethod') or '?'}"
__all__ = ["request_params", "verdict"] __all__ = ["request_params", "verdicts"]
-21
View File
@@ -13,7 +13,6 @@ from __future__ import annotations
import tomllib import tomllib
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Literal
FAMILY_IDS = ( FAMILY_IDS = (
"injection", "injection",
@@ -28,7 +27,6 @@ FAMILY_IDS = (
"unknown", "unknown",
) )
VERIFIER_KINDS = ("canary", "static", "none") VERIFIER_KINDS = ("canary", "static", "none")
Relation = Literal["same", "parent_child", "lateral", "fabricated"]
DEFAULT_FAMILIES_PATH = Path(__file__).resolve().parents[1] / "ruleset" / "families.toml" DEFAULT_FAMILIES_PATH = Path(__file__).resolve().parents[1] / "ruleset" / "families.toml"
_FAMILY_FIELDS = frozenset({"id", "description", "cwes", "mechanisms", "verifier", "parent", "limits"}) _FAMILY_FIELDS = frozenset({"id", "description", "cwes", "mechanisms", "verifier", "parent", "limits"})
@@ -72,24 +70,6 @@ class FamilyTaxonomy:
def family_of_mechanism(self, mechanism: str) -> Family | None: def family_of_mechanism(self, mechanism: str) -> Family | None:
return next((family for family in self.families if mechanism in family.mechanisms), None) return next((family for family in self.families if mechanism in family.mechanisms), None)
@property
def hierarchical(self) -> bool:
"""True when some family declares a parent. While this is False, partial credit can never be earned."""
return any(family.parent for family in self.families)
def related(self, left: str, right: str) -> Relation:
"""How two family answers relate. Abstention is not partial credit: ``unknown`` against a real
family is ``lateral``, never ``parent_child``."""
known = {family.id: family for family in self.families}
if left not in known or right not in known:
return "fabricated"
if left == right:
return "same"
if known[left].parent == right or known[right].parent == left:
return "parent_child"
return "lateral"
def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy: def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
"""Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file); """Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file);
a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``.""" a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``."""
@@ -189,7 +169,6 @@ __all__ = [
"FamiliesError", "FamiliesError",
"Family", "Family",
"FamilyTaxonomy", "FamilyTaxonomy",
"Relation",
"load_families", "load_families",
] ]
-6
View File
@@ -13,7 +13,6 @@ Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
runs/<run>/<task>/output.tar the runner's PUT runs/<run>/<task>/output.tar the runner's PUT
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
runs/<run>/<task>/done.json status, bytes, file count, finished runs/<run>/<task>/done.json status, bytes, file count, finished
runs/<run>/inputs/<name> an input no task produces (``put_input``)
Nothing here logs or prints a URL. Nothing here logs or prints a URL.
""" """
@@ -164,11 +163,6 @@ class Delivery:
put.append(artifact) put.append(artifact)
return put return put
def put_input(self, name: str, path: Path) -> str:
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
# --- state --------------------------------------------------------------------------------------------------- # --- state ---------------------------------------------------------------------------------------------------
def save_state(self, data: Mapping[str, Any]) -> None: def save_state(self, data: Mapping[str, Any]) -> None:
+5 -168
View File
@@ -1,61 +1,29 @@
"""Per-task network egress on Agent Substrate (ai-service-plane, task 0: gateway + receiver Service). """Per-task TLS egress to the store, Git hosts and model endpoints.
Every TCP connection an actor opens (DNS aside) is redirected to Substrate's egress gateway The gateway denies by default. Egress writes and checks each actor's policy before resume,
(``atenet-egress.ate-system``), which denies by default: an actor without an ``EgressPolicy`` gets no tunnel, and then waits for the cached denial to expire. Deleting the actor removes its policy.
a refused connection is accepted and then closed. :func:`policy_for` grants a task exactly what it needs:
* ``http`` on port 80 to the artifact receiver's cluster name (``ousast-receiver.ax-system.svc.cluster.local``, a
selector-less Service whose endpoint is this host's receiver, ``ops/ax/receiver-service.yaml.tmpl``). The actor
cannot resolve cluster names, so the runner dials ``OUSAST_ARTIFACT_DIAL`` (the Service's ClusterIP, port 80)
with the receiver's name as ``Host``. The gateway decides the policy on that ``Host`` but connects to the
address the actor dialled (measured live: agentgateway's upstream is the CONNECT authority, not the Host), so
the dial address must be one the gateway pod reaches: the ClusterIP, which kube-proxy maps to the host.
* ``tls_passthrough`` on 443 for the Git hosts of the bound Workspaces' ``https`` repos (smart HTTP talks to the
repo's host only) and for the bound Model's ``openultrasast.io/egress-hosts`` (``api.deepseek.com``).
Nothing else: no wildcard, no IP address (the gateway rejects them), no ``https`` rule (that one intercepts TLS and
the actor would have to trust the gateway CA, which ax templates do not mount). :class:`Egress` writes the policy
with ``kubectl ate create egress-policy`` after ``ax apply`` created the actor, reads it back, and before resume
waits ``OUSAST_EGRESS_SETTLE`` s (11) because the gateway caches a missing policy as deny for 10 s. The kubectl-ate
CLI has no delete verb: Substrate's store deletes the policy with its actor (``ON DELETE CASCADE``), so
:meth:`Egress.delete` runs after ``ax delete task`` and confirms the policy is gone.
""" """
from __future__ import annotations from __future__ import annotations
import io
import ipaddress import ipaddress
import json import json
import os import os
import shutil
import subprocess import subprocess
import tarfile
import threading
import time import time
from collections.abc import Iterable, Mapping from collections.abc import Iterable, Mapping
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any from typing import Any
from urllib.parse import unquote, urlsplit from urllib.parse import urlsplit
from .manifests import Model, Task, Workspace from .manifests import Model, Task, Workspace
__all__ = [ __all__ = [
"INPUTS_PATH",
"RECEIVER_HOST",
"RECEIVER_PORT",
"Egress", "Egress",
"Receiver",
"git_hosts", "git_hosts",
"receiver_cluster_ip",
"policy_for", "policy_for",
"receiver_address",
"rules_of", "rules_of",
] ]
RECEIVER_HOST = "ousast-receiver.ax-system.svc.cluster.local"
INPUTS_PATH = "/inputs/" # GET <receiver>/inputs/<producer>/<artifact>: a consumer fetches a declared input
RECEIVER_PORT = 18090 # the receiver Service's targetPort; ops/ax/up.sh renders the Service with the same number
_NOT_FOUND = ("not found", "notfound") _NOT_FOUND = ("not found", "notfound")
@@ -89,12 +57,10 @@ def policy_for(
workspaces: Iterable[Workspace], workspaces: Iterable[Workspace],
model: Model | None, model: Model | None,
store_host: str | None, store_host: str | None,
receiver_host: str | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over """The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. An http Git repo cannot be reached under
``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
hostnames only).""" hostnames only)."""
tls, plain = git_hosts(workspaces) tls, plain = git_hosts(workspaces)
@@ -106,8 +72,6 @@ def policy_for(
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname") raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
hosts.add(store_host.lower()) hosts.add(store_host.lower())
rules: list[dict[str, Any]] = [] rules: list[dict[str, Any]] = []
if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it)
rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}})
if hosts: if hosts:
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}}) rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
return {"rules": rules} return {"rules": rules}
@@ -125,41 +89,6 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]:
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True)) return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
def receiver_cluster_ip(context: str) -> str | None:
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when
kubectl cannot say."""
if not context:
raise RuntimeError("receiver_cluster_ip needs the profile's kube context")
argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"]
try:
proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20)
except (OSError, subprocess.TimeoutExpired):
return None
address = proc.stdout.strip()
return address if proc.returncode == 0 and _is_ip(address) else None
def receiver_address(port: int, context: str | None) -> tuple[str, str | None]:
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the
profile's kube context the Service's ClusterIP is looked up in.
``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://<host>:<port>/`` and no dial (tests, a host the actor
reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the
Service's ClusterIP on port 80: the gateway checks the policy against ``Host`` but connects to the address the
actor dialled (its CONNECT authority), so that address must reach the receiver from the gateway pod.
"""
host = os.environ.get("OUSAST_ARTIFACT_HOST")
if host:
return f"http://{host}:{port}/", None
dial = os.environ.get("OUSAST_ARTIFACT_DIAL")
if not dial:
if not context:
raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up")
address = receiver_cluster_ip(context)
dial = f"{address}:80" if address else None
return f"http://{RECEIVER_HOST}/", dial
class Egress: class Egress:
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake.""" """``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
@@ -230,95 +159,3 @@ class Egress:
time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11)) time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11))
class Receiver(ThreadingHTTPServer):
"""Accepts one tar per running task and extracts it under ``<run dir>/<task>/`` (POST); serves a running task
the producer artifacts it declared as inputs (``GET /inputs/<producer>/<artifact>``, ``inputs[task]``)."""
daemon_threads = True
def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None:
super().__init__(("0.0.0.0", port), _Handler)
self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]()
self.inputs: dict[str, frozenset[str]] = {}
self.lock = threading.Lock()
self.url, self.dial = receiver_address(self.server_address[1], context)
def _task(self, headers: Any) -> str | None:
"""The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None."""
task = str(headers.get("X-Ousast-Task", ""))
with self.lock:
return task if headers.get("X-Ousast-Run") == self.run_name and task in self.running else None
def input_file(self, headers: Any, ref: str) -> tuple[int, str | Path]:
"""(200, the file) when ``ref`` is one of the requesting task's declared inputs, else (403/404, why).
``ref`` must equal a declared ``<producer>/<artifact>`` exactly and resolve to a regular file under the run
directory; nothing is normalised first, so ``..`` or an absolute path never names a declared input."""
task = self._task(headers)
if task is None:
return 403, "unknown run or task"
with self.lock:
declared = ref in self.inputs.get(task, frozenset())
if not declared:
return 403, f"{ref!r} is not an input of {task}"
root = self.base.resolve()
path = (root / ref).resolve()
if not path.is_relative_to(root) or not path.is_file():
return 404, f"{ref} was not delivered"
return 200, path
def accept(self, headers: Any, body: bytes) -> tuple[int, str]:
task = self._task(headers)
if task is None:
return 403, "unknown run or task"
if not str(headers.get("Content-Type", "")).startswith("application/x-tar"):
return 415, "expected application/x-tar"
dest = (self.base / task).resolve()
try:
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as tar:
members = tar.getmembers()
for m in members:
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
return 400, f"rejected member {m.name}"
dest.mkdir(parents=True, exist_ok=True)
tar.extractall(dest, members) # every member checked above
except tarfile.TarError as exc:
return 400, f"bad tar: {exc}"
with self.lock:
self.delivered.add(task)
return 200, "ok"
class _Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802 - http.server's naming
server: Receiver = self.server # type: ignore[assignment]
path = urlsplit(self.path).path
if not path.startswith(INPUTS_PATH):
self._text(404, "not found")
return
code, found = server.input_file(self.headers, unquote(path[len(INPUTS_PATH) :]))
if not isinstance(found, Path):
self._text(code, found)
return
with found.open("rb") as source: # streamed: a producer artifact can be far larger than any env value
self.send_response(200)
self.send_header("Content-Type", "application/octet-stream")
self.send_header("Content-Length", str(os.fstat(source.fileno()).st_size))
self.end_headers()
shutil.copyfileobj(source, self.wfile, 1 << 16)
def do_POST(self) -> None: # noqa: N802 - http.server's naming
server: Receiver = self.server # type: ignore[assignment]
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
self._text(*server.accept(self.headers, body))
def _text(self, code: int, text: str) -> None:
data = text.encode()
self.send_response(code)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
def log_message(self, format: str, *args: object) -> None:
return
+11 -33
View File
@@ -10,8 +10,7 @@ credential from this process's environment, never from a file (Req 4.6), and the
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the ``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind delivery fail it; ``ax delete task`` follows.
profile for group 2 only (task 2.6 removes it).
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
@@ -40,13 +39,12 @@ from dataclasses import dataclass, field
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from urllib.parse import urlsplit
import yaml import yaml
from .delivery import Delivery, DeliveryError, declared from .delivery import Delivery, DeliveryError, declared
from .doctor import doctor from .doctor import doctor
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for from .egress import Egress, policy_for
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
from .memory import open_store from .memory import open_store
from .profile import PlaneProfile, load_profile from .profile import PlaneProfile, load_profile
@@ -142,16 +140,14 @@ def _model_doc(model: Model) -> dict[str, Any]:
return _doc("Model", model.metadata.name, spec, model.metadata.atespace) return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]: def render_task(run: Run, entry: RunTask, manifests: Manifests) -> list[dict[str, Any]]:
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced """The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
(``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given.""" (``OUSAST_INPUTS``) after the start by presigned URL."""
task: Task = manifests.tasks[entry.task] task: Task = manifests.tasks[entry.task]
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT) ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
env = {e.name: e.value for e in task.env} env = {e.name: e.value for e in task.env}
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}") env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
budget = _fields(entry.budget) if entry.budget else {} budget = _fields(entry.budget) if entry.budget else {}
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()}) env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()} inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
@@ -341,25 +337,22 @@ def _execute(
base: Path, base: Path,
delivery: Delivery, delivery: Delivery,
router: Router, router: Router,
receiver: Receiver | None = None,
) -> tuple[str, str]: ) -> tuple[str, str]:
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task] ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml" atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
manifest.parent.mkdir(parents=True, exist_ok=True) manifest.parent.mkdir(parents=True, exist_ok=True)
docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None) docs = render_task(run, entry, manifests)
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8") manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "") model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
workspaces = [manifests.workspaces[b.name] for b in task.workspaces] workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body body = delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None)
if receiver is None:
delivery.reset(entry.name) # a previous attempt's tar must not complete this one delivery.reset(entry.name) # a previous attempt's tar must not complete this one
def delivered() -> bool: def delivered() -> bool:
return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name) return delivery.delivered(entry.name)
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
policy = policy_for(task, workspaces, model, store_host, receiver_host) policy = policy_for(task, workspaces, model, delivery.host)
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body) start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
egress.apply(ax_name, atespace, policy) egress.apply(ax_name, atespace, policy)
@@ -369,7 +362,7 @@ def _execute(
return "failed", str(exc) return "failed", str(exc)
try: try:
failure = _await(ax, ax_name, delivered, start) failure = _await(ax, ax_name, delivered, start)
if failure is None and receiver is None: if failure is None:
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
except DeliveryError as exc: except DeliveryError as exc:
failure = str(exc) failure = str(exc)
@@ -408,11 +401,6 @@ def run(
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}}, _read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
mirror=delivery.save_state, mirror=delivery.save_state,
) )
receiver: Receiver | None = None
if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it)
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
threading.Thread(target=receiver.serve_forever, daemon=True).start()
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]() cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"] pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
for entry in run_spec.tasks: for entry in run_spec.tasks:
@@ -427,7 +415,7 @@ def run(
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
state.set(entry.name, status="running", started=_now(), finished=None) state.set(entry.name, status="running", started=_now(), finished=None)
try: try:
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver) outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router)
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
outcome = ("failed", f"{type(exc).__name__}: {exc}") outcome = ("failed", f"{type(exc).__name__}: {exc}")
finished.put((entry.name, *outcome)) finished.put((entry.name, *outcome))
@@ -444,27 +432,17 @@ def run(
pending.remove(entry) pending.remove(entry)
running[entry.name] = entry running[entry.name] = entry
blocked.add(entry.serialize or "") blocked.add(entry.serialize or "")
if receiver is not None:
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
receiver.running.add(entry.name)
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
threading.Thread(target=worker, args=(entry,), daemon=True).start() threading.Thread(target=worker, args=(entry,), daemon=True).start()
if not running: if not running:
break # nothing can start: a producer stopped short break # nothing can start: a producer stopped short
name, status, reason = finished.get() name, status, reason = finished.get()
del running[name] del running[name]
if receiver is not None:
with receiver.lock:
receiver.running.discard(name)
receiver.inputs.pop(name, None)
summary = _read_json(base / name / "summary.json") or {} summary = _read_json(base / name / "summary.json") or {}
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None) state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
halted = halted or status == "failed" halted = halted or status == "failed"
if status == "unfinished": if status == "unfinished":
pending = [t for t in pending if name not in t.producers] pending = [t for t in pending if name not in t.producers]
finally: finally:
if receiver is not None:
receiver.shutdown()
lock.unlink(missing_ok=True) lock.unlink(missing_ok=True)
statuses = [state.status(t.name) for t in run_spec.tasks] statuses = [state.status(t.name) for t in run_spec.tasks]
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished" result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
@@ -546,4 +524,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No
return "\n".join(lines) return "\n".join(lines)
__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"] __all__ = ["Ax", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
+18 -52
View File
@@ -25,8 +25,7 @@ The command runs as a child process, ``python -m openultrasast.plane.tasks.<spec
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail), runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` (or POSTs it to ``OUSAST_ARTIFACT_URL``, the receiver PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` -- that delivery is the task's
path kept until task 2.6 removes it) -- that delivery is the task's
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM, ``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
@@ -48,13 +47,9 @@ Environment the runner reads, all optional except the two ax variables:
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10 OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout) OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
OUSAST_GIT_PINS JSON ``{"<workspace>/<git name>": "<40-hex commit>"}`` from the reconciler OUSAST_GIT_PINS JSON ``{"<workspace>/<git name>": "<40-hex commit>"}`` from the reconciler
OUSAST_ARTIFACT_URL, OUSAST_RUN where to deliver, and the run name carried in ``X-Ousast-Run``
OUSAST_ARTIFACT_DIAL ``host:port`` to connect to instead of the URL's host, which is sent as
``Host`` (the egress gateway decides on and resolves that name)
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
command each is fetched from ``delivery.inputs[NAME]`` (else from command each is fetched from ``delivery.inputs[NAME]``
``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` with the receiver's dial and headers)
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
fetch delivers a failed summary naming the input, the command never runs fetch delivers a failed summary naming the input, the command never runs
@@ -700,16 +695,6 @@ def build_tar(output_dir: Path) -> bytes:
return buffer.getvalue() return buffer.getvalue()
def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[str, dict[str, str]]:
"""(URL to connect to, headers): with ``dial`` the connection goes there and the URL's host is sent as ``Host``."""
out = dict(headers)
if not dial:
return url, out
parts = urlsplit(url)
out["Host"] = parts.netloc
return parts._replace(netloc=dial).geturl(), out
def safe_url(url: str) -> str: def safe_url(url: str) -> str:
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential).""" """``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
parts = urlsplit(url) parts = urlsplit(url)
@@ -723,18 +708,14 @@ def deliver(
*, *,
attempts: int = DELIVERY_ATTEMPTS, attempts: int = DELIVERY_ATTEMPTS,
backoff: float = 1.0, backoff: float = 1.0,
dial: str | None = None,
method: str = "POST",
) -> int: ) -> int:
"""Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry """PUT the payload with retries and doubling backoff; return the 2xx status.
with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query.
``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it Log lines and errors carry the URL without its query.
cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself.
""" """
last, (target, headers) = "no attempt made", _dialled(url, headers, dial) last = "no attempt made"
for attempt in range(1, attempts + 1): for attempt in range(1, attempts + 1):
request = urllib.request.Request(target, data=payload, method=method, headers=headers) request = urllib.request.Request(url, data=payload, method="PUT", headers=headers)
try: try:
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
status = int(response.status) status = int(response.status)
@@ -752,16 +733,16 @@ def deliver(
def fetch( def fetch(
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0, dial: str | None = None url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0
) -> int: ) -> int:
"""GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`. """GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`.
Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed.""" Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed."""
last, (target, headers) = "no attempt made", _dialled(url, headers, dial) last = "no attempt made"
dest.parent.mkdir(parents=True, exist_ok=True) dest.parent.mkdir(parents=True, exist_ok=True)
partial = dest.with_name(dest.name + ".part") partial = dest.with_name(dest.name + ".part")
for attempt in range(1, attempts + 1): for attempt in range(1, attempts + 1):
request = urllib.request.Request(target, method="GET", headers=headers) request = urllib.request.Request(url, method="GET", headers=headers)
try: try:
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310 with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310
shutil.copyfileobj(response, out, 1 << 16) shutil.copyfileobj(response, out, 1 << 16)
@@ -784,7 +765,7 @@ def fetch(
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]: def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a """Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
presigned GET, no headers) when the start request carries one, else from the receiver. presigned GET, no headers) carried by the start request.
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing.""" :class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
raw = environ.get("OUSAST_INPUTS") raw = environ.get("OUSAST_INPUTS")
@@ -796,21 +777,15 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None
if not isinstance(inputs, dict): if not isinstance(inputs, dict):
raise RunnerError("OUSAST_INPUTS must map input names to <producer>/<artifact>") raise RunnerError("OUSAST_INPUTS must map input names to <producer>/<artifact>")
url = environ.get("OUSAST_ARTIFACT_URL")
headers = {"X-Ousast-Run": environ.get("OUSAST_RUN", ""), "X-Ousast-Task": environ.get("OUSAST_TASK", "")}
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0")) backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
fetched: list[Path] = [] fetched: list[Path] = []
for name, ref in sorted(inputs.items()): for name, ref in sorted(inputs.items()):
dest = environ.get(f"OUSAST_INPUT_{name}") dest = environ.get(f"OUSAST_INPUT_{name}")
presigned = delivery.inputs.get(name) if delivery is not None else None presigned = delivery.inputs.get(name) if delivery is not None else None
if not dest or not isinstance(ref, str) or not (presigned or url): if not dest or not isinstance(ref, str) or not presigned:
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}") raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL or OUSAST_INPUT_{name}")
try: try:
if presigned:
size = fetch(presigned, {}, Path(dest), backoff=backoff) size = fetch(presigned, {}, Path(dest), backoff=backoff)
else:
source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
except DeliveryError as exc: except DeliveryError as exc:
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest) log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
@@ -818,23 +793,14 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
return fetched return fetched
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool: def _deliver_output(output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
"""PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver; """PUT the output tar to ``delivery.put`` when the start request carried one;
False when there is nowhere to deliver; :class:`DeliveryError` when it fails.""" False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0")) backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
if delivery is not None and delivery.put: if delivery is not None and delivery.put:
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT") deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff)
return True return True
url = environ.get("OUSAST_ARTIFACT_URL")
if not url:
return False return False
# the Run's task name (``OUSAST_TASK``) is what the receiver knows; the ax Task is named ``<run>-<task>``
headers = {"Content-Type": "application/x-tar", "X-Ousast-Task": environ.get("OUSAST_TASK") or task.metadata.name}
run = environ.get("OUSAST_RUN")
if run:
headers["X-Ousast-Run"] = run
deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
return True
# --- entrypoint -------------------------------------------------------------------------------------------------- # --- entrypoint --------------------------------------------------------------------------------------------------
@@ -849,7 +815,7 @@ class _Session:
@property @property
def marker(self) -> Path: def marker(self) -> Path:
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the receiver has it.""" """``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the store has it."""
return self.state_dir / f"{self.task.metadata.name}.done.json" return self.state_dir / f"{self.task.metadata.name}.done.json"
@property @property
@@ -893,7 +859,7 @@ def _finish(
""" """
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)} marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
try: try:
marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery) marker["delivered"] = _deliver_output(session.output_dir, env, delivery)
except DeliveryError as exc: except DeliveryError as exc:
log.error("artifact delivery failed: %s", exc) log.error("artifact delivery failed: %s", exc)
marker.update(status="failed", reason=f"artifact delivery failed: {exc}") marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
@@ -938,7 +904,7 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
return 0 return 0
try: try:
_materialise_once(session, env) _materialise_once(session, env)
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only) fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it
except RunnerError as exc: except RunnerError as exc:
log.error("runner failed: %s", exc) log.error("runner failed: %s", exc)
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir)) failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
+1 -17
View File
@@ -199,22 +199,6 @@ def _model(summary: object) -> str | None:
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]:
""":func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present)."""
final = base / f"{case}-final" / "agreed.json"
return case_features(
facts=_json(base / f"{case}-facts" / "facts.json"),
passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES},
models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES},
agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"),
alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"),
alerts_summary=_json(base / f"{case}-alerts" / "summary.json"),
workspace=workspace,
facts_summary=_json(base / f"{case}-facts" / "summary.json"),
model_roles=_json(base / f"{case}-roles" / "roles.json"),
)
def _path(env: Mapping[str, str], name: str) -> Path | None: def _path(env: Mapping[str, str], name: str) -> Path | None:
return Path(env[name]) if env.get(name) else None return Path(env[name]) if env.get(name) else None
@@ -266,7 +250,7 @@ def main(environ: Mapping[str, str] | None = None) -> int:
return 0 if summary["status"] == "done" else 2 return 0 if summary["status"] == "done" else 2
__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"] __all__ = ["DROPPED_ON_READ", "case_features", "main", ]
if __name__ == "__main__": if __name__ == "__main__":
+1 -51
View File
@@ -4,7 +4,7 @@ import json
from collections.abc import Collection from collections.abc import Collection
from dataclasses import asdict, dataclass, replace from dataclasses import asdict, dataclass, replace
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Protocol from typing import TYPE_CHECKING
from .preprocess import FileTarget from .preprocess import FileTarget
@@ -28,11 +28,6 @@ class RankingScore:
static_boosts: list[str] static_boosts: list[str]
class RankerClient(Protocol):
def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object:
raise NotImplementedError
def rank_targets(targets: list[FileTarget], model_rankings: object | None = None, model_id: str | None = None) -> list[RankingScore]: def rank_targets(targets: list[FileTarget], model_rankings: object | None = None, model_id: str | None = None) -> list[RankingScore]:
model_by_path = parse_ranker_response(model_rankings, model_id) if model_rankings is not None else {} model_by_path = parse_ranker_response(model_rankings, model_id) if model_rankings is not None else {}
rankings: list[RankingScore] = [] rankings: list[RankingScore] = []
@@ -43,32 +38,6 @@ def rank_targets(targets: list[FileTarget], model_rankings: object | None = None
return sorted(rankings, key=lambda item: (-item.priority, item.path)) return sorted(rankings, key=lambda item: (-item.priority, item.path))
def rank_targets_with_model(
targets: list[FileTarget],
*,
client: RankerClient,
model: str,
chunk_size: int = 150,
timeout_seconds: int = 60,
) -> list[RankingScore]:
if chunk_size < 1:
raise RankingError("chunk_size must be at least 1")
model_rankings: list[dict[str, object]] = []
for start in range(0, len(targets), chunk_size):
chunk = targets[start : start + chunk_size]
response = client.complete_json(
model=model,
messages=[
{"role": "system", "content": _ranker_system_prompt()},
{"role": "user", "content": json.dumps([_target_prompt_payload(target) for target in chunk], sort_keys=True)},
],
timeout_seconds=timeout_seconds,
)
for ranking in parse_ranker_response(response, model).values():
model_rankings.append(asdict(ranking))
return rank_targets(targets, {"rankings": model_rankings}, model_id=model)
def write_rankings(rankings: list[RankingScore], path: Path) -> None: def write_rankings(rankings: list[RankingScore], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
payload = {"rankings": [asdict(ranking) for ranking in rankings]} payload = {"rankings": [asdict(ranking) for ranking in rankings]}
@@ -205,25 +174,6 @@ def _clamp_score(value: int) -> int:
return min(5, max(1, value)) return min(5, max(1, value))
def _ranker_system_prompt() -> str:
return (
"Score each file for security review priority. Return JSON only with a rankings array. "
"Each item must include path, surface, influence, reachability, rationale, and optional static_boosts. "
"Scores must be integers from 1 to 5."
)
def _target_prompt_payload(target: FileTarget) -> dict[str, object]:
return {
"path": target.path,
"language": target.language,
"loc": target.loc,
"tags": target.tags,
"has_fuzz_entry_point": target.has_fuzz_entry_point,
"reachability_hints": target.reachability_hints,
}
def _reachability_from_hints(hints: list[dict[str, object]]) -> int: def _reachability_from_hints(hints: list[dict[str, object]]) -> int:
score = 0 score = 0
for hint in hints: for hint in hints:
-82
View File
@@ -7,20 +7,14 @@ call is conservatively charged its reservation on resume, never silently free.
from __future__ import annotations from __future__ import annotations
import argparse
import hashlib
import json
import re import re
import time import time
from collections import Counter from collections import Counter
from collections.abc import Callable from collections.abc import Callable
from dataclasses import asdict, dataclass, replace from dataclasses import asdict, dataclass, replace
from importlib import import_module
from pathlib import Path
from typing import Any, Protocol from typing import Any, Protocol
from ..plane.budget import BudgetExhausted, SpendBudget from ..plane.budget import BudgetExhausted, SpendBudget
from ..plane.memory import open_store
from .board import Board from .board import Board
from .budget import SearchBudget from .budget import SearchBudget
@@ -421,79 +415,3 @@ class Coordinator:
@staticmethod @staticmethod
def _refs(refs: Any) -> bool: def _refs(refs: Any) -> bool:
return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs) return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs)
class BatchExecutor:
"""Adapter to the existing AX/VM transfer and checkpoint protocol.
Pass AXLane/DockerLane instances and a trusted worker command. No default
worker is invented here: reason/explore entrypoints belong to group 5.
The coordinator owns retries, so batch performs exactly one attempt.
"""
def __init__(self, *, image: str, command: tuple[str, ...], output: Path, lanes: dict[str, Any]) -> None:
batch = import_module("benchmarks.ax.batch")
batch.validate_image(image)
if not command:
raise ValueError("worker command required")
self.image, self.command, self.output, self.lanes = image, command, output, lanes
def __call__(self, task: SearchTask) -> dict[str, Any]:
import copy
batch = import_module("benchmarks.ax.batch")
if task.lane not in self.lanes:
return {"status": "sandbox_failure", "cost_usd": 0}
data = json.dumps(asdict(task), sort_keys=True, allow_nan=False).encode()
digest = hashlib.sha256(data).hexdigest()
identity = task.search_id + "-" + task.id
item = batch.Item(
identity,
{"SPEC_URL": data},
"result.dat",
self.command,
{"SEARCH_STEP": task.step, "SEARCH_ID": task.search_id, "SEARCH_TASK_ID": task.id},
digest,
)
def validate(raw: bytes, output: Path) -> None:
if len(raw) > 262144:
raise ValueError("search result size exceeded")
record = json.loads(raw)
if not isinstance(record, dict) or record.get("status") not in (
"ok",
"sandbox_failure",
"instrument_failure",
"oom",
"execution_failure",
):
raise ValueError("invalid search result")
(output / "result.json").write_text(json.dumps(record))
workload = batch.Workload(self.image, frozenset({"SPEC_URL", "RESULT_URL"}), task.limits["task_wall_seconds"], validate)
lane = copy.copy(self.lanes[task.lane])
if hasattr(lane, "workload"):
lane.workload = workload
out = self.output / task.search_id / task.id
batch.dispatch([item], workload, out, {task.lane: lane}, lanes=(task.lane,), max_attempts=1)
record_path = out / (hashlib.sha256(identity.encode()).hexdigest() + ".json")
if not record_path.exists():
return {"status": "execution_failure", "cost_usd": 0}
record: dict[str, Any] = json.loads(record_path.read_text())
return record
def main() -> None:
"""Offline checkpoint inspection, also useful before resuming with an executor."""
parser = argparse.ArgumentParser(description="Inspect a search board checkpoint (no model calls)")
parser.add_argument("--memory", required=True)
parser.add_argument("--search-id", required=True)
parser.add_argument("--head", required=True)
args = parser.parse_args()
board = Board.resume(open_store(args.memory), args.search_id, args.head, coordinator="inspect")
print(board.snapshot(), end="")
if __name__ == "__main__":
main()
+1 -18
View File
@@ -22,7 +22,7 @@ import zipfile
from collections.abc import Callable from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from contextlib import suppress from contextlib import suppress
from dataclasses import asdict, dataclass, field, replace from dataclasses import dataclass, field, replace
from pathlib import Path from pathlib import Path
from typing import Any, cast from typing import Any, cast
@@ -709,20 +709,3 @@ def verify(
isolation_mode=mode, isolation_mode=mode,
chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)), chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)),
) )
def main() -> None:
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("affected", type=Path)
parser.add_argument("safe", type=Path)
parser.add_argument("artefact", type=Path)
parser.add_argument("family")
args = parser.parse_args()
result = verify(Side(args.affected), Side(args.safe), args.artefact, args.family)
print(json.dumps(asdict(result), indent=2))
if __name__ == "__main__":
main()
+1 -45
View File
@@ -2,11 +2,9 @@
from __future__ import annotations from __future__ import annotations
import argparse
import copy import copy
import json import json
import math import math
import os
import time import time
from pathlib import Path from pathlib import Path
from typing import Any, Protocol, cast from typing import Any, Protocol, cast
@@ -15,12 +13,11 @@ import yaml
from ..model.endpoint import Prices from ..model.endpoint import Prices
from ..plane.budget import BudgetExhausted, SpendBudget, cost_of from ..plane.budget import BudgetExhausted, SpendBudget, cost_of
from ..provider.openrouter import OpenRouterChatClient, parse_json_content from ..provider.openrouter import parse_json_content
from . import _sandbox from . import _sandbox
from .budget import SearchBudget from .budget import SearchBudget
from .coordinator import SearchTask from .coordinator import SearchTask
from .demo import DEMO_SCHEMA, FAMILY_ORACLES, MAX_DEMO_BYTES, load_demo, validate_demo, validate_oracle from .demo import DEMO_SCHEMA, FAMILY_ORACLES, MAX_DEMO_BYTES, load_demo, validate_demo, validate_oracle
from .executor import ObjectStoreExecutor
from .task_storage import exception_reason from .task_storage import exception_reason
@@ -525,44 +522,3 @@ class Worker:
"cost_usd": meter.spent, "cost_usd": meter.spent,
"isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"), "isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
} }
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--task", required=True, type=Path)
parser.add_argument("--repo", required=True, type=Path)
parser.add_argument("--demo", required=True, type=Path)
parser.add_argument("--model", required=True)
parser.add_argument("--input-per-m", required=True, type=float)
parser.add_argument("--output-per-m", required=True, type=float)
parser.add_argument("--command-put-url")
parser.add_argument("--result-get-url")
args = parser.parse_args()
if bool(args.command_put_url) != bool(args.result_get_url):
parser.error("both executor mailbox URLs are required")
executor = ObjectStoreExecutor(args.command_put_url, args.result_get_url) if args.command_put_url else None
task = SearchTask(**json.loads(args.task.read_text()))
if task.step == "explore" and executor is None:
parser.error("explore requires a separate repository executor")
client = OpenRouterChatClient(
api_key=os.environ["OPENROUTER_API_KEY"],
base_url=os.environ.get("OPENROUTER_BASE_URL", OpenRouterChatClient.base_url),
max_attempts=1,
)
worker = Worker(
client,
repo=args.repo,
demo=args.demo,
model=args.model,
prices=Prices(args.input_per_m, args.input_per_m, args.output_per_m),
executor=executor,
)
try:
print(json.dumps(worker(task)))
finally:
if executor is not None:
executor.submit("stop", {}, timeout_seconds=10)
if __name__ == "__main__":
main()
+4 -78
View File
@@ -15,12 +15,8 @@ from .findings import StaticFinding
from .hunter_tools import ( from .hunter_tools import (
PathEscapesRepo, PathEscapesRepo,
clamp_repo_path, clamp_repo_path,
entry_points,
find_refs, find_refs,
flows,
grep_repo, grep_repo,
obligations,
read_definition,
read_file, read_file,
) )
from .provider.openrouter import OpenRouterChatClient, OpenRouterError from .provider.openrouter import OpenRouterChatClient, OpenRouterError
@@ -31,7 +27,6 @@ DEFAULT_MAX_STEPS = 4
CLIENT_ENV = "OPENULTRASAST_HUNTER_CLIENT" CLIENT_ENV = "OPENULTRASAST_HUNTER_CLIENT"
_FINDING_ID_PREFIX = "tool-hunter:" _FINDING_ID_PREFIX = "tool-hunter:"
DEFAULT_TOOL_NAMES = ("read_file", "grep_repo", "find_refs") DEFAULT_TOOL_NAMES = ("read_file", "grep_repo", "find_refs")
_HUNTER_TOOL_NAMES = frozenset(DEFAULT_TOOL_NAMES) | {"read_definition", "entry_points", "flows", "obligations"}
_SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"}) _SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"})
_SCRIPTED_FLAGS = frozenset({"scripted", "script"}) _SCRIPTED_FLAGS = frozenset({"scripted", "script"})
_DUMP_FLAGS = frozenset({"dump", "dump-only"}) _DUMP_FLAGS = frozenset({"dump", "dump-only"})
@@ -91,50 +86,6 @@ HUNTER_TOOLS: list[dict[str, object]] = [
}, },
}, },
}, },
{
"type": "function",
"function": {
"name": "read_definition",
"description": "Resolve one named function to its file, line span and body. Use it instead of guessing what a callee does.",
"parameters": {
"type": "object",
"properties": {"symbol": {"type": "string"}, "path": {"type": "string"}, "max_chars": {"type": "integer"}},
"required": ["symbol"],
},
},
},
{
"type": "function",
"function": {
"name": "entry_points",
"description": "List the handlers this repository exposes, with their access classification. Identifiers and line spans only.",
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": []},
},
},
{
"type": "function",
"function": {
"name": "flows",
"description": "List adjudicated source-to-sink records inside one function. Identifiers and line numbers only.",
"parameters": {
"type": "object",
"properties": {"path": {"type": "string"}, "function": {"type": "string"}},
"required": ["path", "function"],
},
},
},
{
"type": "function",
"function": {
"name": "obligations",
"description": "List what one function owes and has not discharged: the operation, the missing discharger and why it is owed.",
"parameters": {
"type": "object",
"properties": {"path": {"type": "string"}, "function": {"type": "string"}},
"required": ["path", "function"],
},
},
},
] ]
_SYSTEM_PROMPT = ( _SYSTEM_PROMPT = (
@@ -330,15 +281,13 @@ def run_tool_hunter(
system_prompt: str = _SYSTEM_PROMPT, system_prompt: str = _SYSTEM_PROMPT,
user_prompt: str | None = None, user_prompt: str | None = None,
context_files: Sequence[str] = (), context_files: Sequence[str] = (),
tools: Sequence[str] | None = None,
max_chars: int = _DEFAULT_MAX_CHARS, max_chars: int = _DEFAULT_MAX_CHARS,
tags: Sequence[str] = ("tool-hunter",), tags: Sequence[str] = ("tool-hunter",),
) -> list[StaticFinding]: ) -> list[StaticFinding]:
"""One bounded hunt. Every keyword defaults to today's behaviour; a per-family detector overrides the """One bounded hunt with a prompt, starting context and finding tags."""
prompt, the context it starts from, the tools it may call and the tags its findings carry."""
if max_steps <= 0: if max_steps <= 0:
return [] return []
schemas = _tool_schemas(tools) schemas = HUNTER_TOOLS
messages: list[dict[str, object]] = [ messages: list[dict[str, object]] = [
{"role": "system", "content": system_prompt}, {"role": "system", "content": system_prompt},
{"role": "user", "content": _user_message(root, hotspots, user_prompt, context_files, max_chars)}, {"role": "user", "content": _user_message(root, hotspots, user_prompt, context_files, max_chars)},
@@ -350,7 +299,7 @@ def run_tool_hunter(
if tool_calls: if tool_calls:
messages.append(_assistant_tool_message(response.content, tool_calls, response.reasoning)) messages.append(_assistant_tool_message(response.content, tool_calls, response.reasoning))
for call in tool_calls: for call in tool_calls:
if call.name in _HUNTER_TOOL_NAMES: if call.name in DEFAULT_TOOL_NAMES:
used_tools = True used_tools = True
messages.append(_tool_result_message(root, call, max_chars)) messages.append(_tool_result_message(root, call, max_chars))
continue continue
@@ -414,12 +363,6 @@ def _looks_like_json(content: str) -> bool:
return stripped.startswith(("[", "{")) return stripped.startswith(("[", "{"))
def _tool_schemas(names: Sequence[str] | None) -> list[dict[str, object]]:
wanted = tuple(names) if names is not None else DEFAULT_TOOL_NAMES
by_name = {str(schema["function"]["name"]): schema for schema in HUNTER_TOOLS} # type: ignore[index]
return [by_name[name] for name in wanted if name in by_name]
def _user_message(root: Path, hotspots: Sequence[Hotspot], user_prompt: str | None, context_files: Sequence[str], max_chars: int) -> str: def _user_message(root: Path, hotspots: Sequence[Hotspot], user_prompt: str | None, context_files: Sequence[str], max_chars: int) -> str:
# Context files FIRST, the question LAST. DeepSeek bills a cached prefix at 1/30 of a fresh one, and the cache # Context files FIRST, the question LAST. DeepSeek bills a cached prefix at 1/30 of a fresh one, and the cache
# is by prefix: with the question first, every candidate in the same file broke the prefix at its first # is by prefix: with the question first, every candidate in the same file broke the prefix at its first
@@ -502,30 +445,13 @@ def _dispatch_tool(root: Path, name: str, arguments: Mapping[str, object], defau
path = _require_str(arguments, "path") path = _require_str(arguments, "path")
max_chars = _optional_int(arguments, "max_chars", default_max_chars) max_chars = _optional_int(arguments, "max_chars", default_max_chars)
return {"path": path, "content": read_file(root, path, max_chars=max_chars)} return {"path": path, "content": read_file(root, path, max_chars=max_chars)}
if name == "read_definition":
symbol = _require_str(arguments, "symbol")
scope = arguments.get("path")
found = read_definition(
root,
symbol,
max_chars=_optional_int(arguments, "max_chars", default_max_chars),
path=str(scope) if isinstance(scope, str) and scope else None,
)
return found if found is not None else {"symbol": symbol, "found": False}
if name == "entry_points":
scope = arguments.get("path")
return entry_points(root, path=str(scope) if isinstance(scope, str) and scope else None)
if name == "flows":
return flows(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function"))
if name == "obligations":
return obligations(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function"))
if name == "grep_repo": if name == "grep_repo":
pattern = _require_str(arguments, "pattern") pattern = _require_str(arguments, "pattern")
max_matches = _optional_int(arguments, "max_matches", _DEFAULT_MAX_MATCHES) max_matches = _optional_int(arguments, "max_matches", _DEFAULT_MAX_MATCHES)
return grep_repo(root, pattern, max_matches=max_matches) return grep_repo(root, pattern, max_matches=max_matches)
if name == "find_refs": if name == "find_refs":
symbol = _require_str(arguments, "symbol") symbol = _require_str(arguments, "symbol")
return find_refs(root, symbol, None) return find_refs(root, symbol)
raise ValueError(f"unknown tool: {name}") raise ValueError(f"unknown tool: {name}")
-10
View File
@@ -36,16 +36,6 @@ class VerificationResult:
context_sources: list[str] context_sources: list[str]
def validate_evidence_transition(current: EvidenceLevel | str, target: EvidenceLevel | str) -> EvidenceLevel:
current_level = EvidenceLevel(current)
target_level = EvidenceLevel(target)
if _EVIDENCE_ORDER[target_level] < _EVIDENCE_ORDER[current_level]:
raise ValueError(f"cannot move evidence level backward: {current_level} -> {target_level}")
if _EVIDENCE_ORDER[target_level] - _EVIDENCE_ORDER[current_level] > 1:
raise ValueError(f"cannot skip evidence levels: {current_level} -> {target_level}")
return target_level
def is_report_verified(evidence_level: EvidenceLevel | str, status: VerificationStatus | str) -> bool: def is_report_verified(evidence_level: EvidenceLevel | str, status: VerificationStatus | str) -> bool:
level = EvidenceLevel(evidence_level) level = EvidenceLevel(evidence_level)
verification_status = VerificationStatus(status) verification_status = VerificationStatus(status)
-29
View File
@@ -6,7 +6,6 @@ from openultrasast.calibration import (
apply_ranking_calibration, apply_ranking_calibration,
calibrate_ranking, calibrate_ranking,
calibrate_rankings, calibrate_rankings,
compute_ranking_metrics,
learning_from_finding_outcome, learning_from_finding_outcome,
record_false_positive_learning, record_false_positive_learning,
write_false_positive_learnings, write_false_positive_learnings,
@@ -62,34 +61,6 @@ def test_false_positive_learning_records_reason_and_scope(tmp_path: Path) -> Non
assert "missing_attacker_control" in output.read_text() assert "missing_attacker_control" in output.read_text()
def test_ranking_metrics_track_false_positive_reasons_by_tier() -> None:
metrics = compute_ranking_metrics(
[
{"tier": "A", "outcome": FindingOutcome.VERIFIED, "time_to_verification_seconds": 10},
{
"tier": "A",
"outcome": FindingOutcome.REJECTED,
"reason": FalsePositiveReason.UNREACHABLE_PATH,
"time_to_verification_seconds": 20,
},
{"tier": "B", "outcome": FindingOutcome.REJECTED, "reason": FalsePositiveReason.DUPLICATE},
{
"tier": "A",
"outcome": FindingOutcome.UNVERIFIED,
"reason": FalsePositiveReason.UNVERIFIED,
"missed_fixture_vulnerabilities": 1,
},
],
tier="A",
)
assert metrics.verified_findings == 1
assert metrics.false_positives == 2
assert metrics.time_to_verification_seconds == 15.0
assert metrics.missed_fixture_vulnerabilities == 1
assert metrics.false_positive_reasons["unreachable_path"] == 1
def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None: def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None:
learning = record_false_positive_learning( learning = record_false_positive_learning(
_finding("auth/login.py"), _finding("auth/login.py"),
+1 -15
View File
@@ -1,7 +1,7 @@
import json import json
from pathlib import Path from pathlib import Path
from openultrasast.findings import build_quick_hunter_prompt, quick_scan_findings, write_findings from openultrasast.findings import quick_scan_findings, write_findings
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
from openultrasast.preprocess import preprocess_repository from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets from openultrasast.rank import rank_targets
@@ -92,17 +92,3 @@ def test_findings_inherit_only_matching_function_level_reachability(tmp_path: Pa
assert unreachable.ranking_priority <= 1.5 assert unreachable.ranking_priority <= 1.5
def test_quick_hunter_prompt_is_bounded() -> None:
repo = Path("/repo/example.py")
target = next(iter(preprocess_repository(repo.parent)[1]), None) if repo.parent.exists() else None
assert target is None
class Target:
path = "example.py"
language = "python"
tags = ["auth_boundary"]
prompt = build_quick_hunter_prompt(Target(), "x" * 5000) # type: ignore[arg-type]
assert "example.py" in prompt
assert len(prompt) < 4300
+1 -15
View File
@@ -6,13 +6,7 @@ from pathlib import Path
import pytest import pytest
from openultrasast.findings import StaticFinding from openultrasast.findings import StaticFinding
from openultrasast.fusion import ( from openultrasast.fusion import FusionDisposition, fuse_finding, fuse_findings, fusion_triggers
FusionDisposition,
fuse_finding,
fuse_findings,
fusion_triggers,
should_fuse,
)
from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus
@@ -63,10 +57,6 @@ def test_triggers_on_high_severity_and_reachable() -> None:
assert "gates_risky_fix" in triggers assert "gates_risky_fix" in triggers
def test_low_severity_unreachable_does_not_trigger() -> None:
assert not should_fuse(_finding(severity="low", reachability="unknown"))
def test_verifier_disagreement_triggers() -> None: def test_verifier_disagreement_triggers() -> None:
triggers = fusion_triggers( triggers = fusion_triggers(
_finding(severity="low", reachability="unknown"), _finding(severity="low", reachability="unknown"),
@@ -83,10 +73,6 @@ def test_evidence_conflict_triggers_when_rejected_but_reachable() -> None:
assert "evidence_conflict" in triggers assert "evidence_conflict" in triggers
def test_high_assurance_request_forces_a_trigger() -> None:
assert should_fuse(_finding(severity="low", reachability="unknown"), high_assurance=True)
# ---- deterministic two-panel decider + dispositions ------------------------- # ---- deterministic two-panel decider + dispositions -------------------------
+4 -68
View File
@@ -1,100 +1,36 @@
import json import json
from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any
import pytest
from openultrasast.config import load_config from openultrasast.config import load_config
from openultrasast.harness import ( from openultrasast.harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
HarnessContractError,
HarnessEvent,
HarnessRuntime,
HarnessTraceWriter,
ProcessorSpec,
write_harness_config,
)
@dataclass def test_runtime_emits_stage_events(tmp_path: Path) -> None:
class RecordingProcessor:
spec: ProcessorSpec
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
return {"seen_stage": event.stage}
@dataclass
class BadWriteProcessor:
spec: ProcessorSpec
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
return {"undeclared": event.event_type}
def test_runtime_emits_stage_events_and_processor_state(tmp_path: Path) -> None:
trace_path = tmp_path / "events.jsonl" trace_path = tmp_path / "events.jsonl"
processor = RecordingProcessor(ProcessorSpec(name="recorder", version="1", handles=("stage_start",), writes=("seen_stage",)))
runtime = HarnessRuntime( runtime = HarnessRuntime(
scan_id="scan-1", scan_id="scan-1",
config=load_config(None), config=load_config(None),
trace_writer=HarnessTraceWriter(trace_path), trace_writer=HarnessTraceWriter(trace_path),
processors=[processor],
) )
result = runtime.run_stage("preprocess", lambda: "ok") result = runtime.run_stage("preprocess", lambda: "ok")
events = [json.loads(line) for line in trace_path.read_text().splitlines()] events = [json.loads(line) for line in trace_path.read_text().splitlines()]
assert result == "ok" assert result == "ok"
assert runtime.state["seen_stage"] == "preprocess" assert [event["event_type"] for event in events] == ["stage_start", "stage_end"]
assert [event["event_type"] for event in events] == ["stage_start", "stage_start", "stage_end"]
assert events[1]["payload"] == {"changed_state": ["seen_stage"], "processor": "recorder"}
def test_strict_contract_mode_rejects_undeclared_writes(tmp_path: Path) -> None: def test_harness_config_serializes_models_and_prompt_hashes(tmp_path: Path) -> None:
processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
runtime = HarnessRuntime(
scan_id="scan-1",
config=load_config(None),
trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
processors=[processor],
)
with pytest.raises(HarnessContractError, match="undeclared writes"):
runtime.emit("stage_start", "rank", {})
def test_warn_contract_mode_records_degradation(tmp_path: Path) -> None:
processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
runtime = HarnessRuntime(
scan_id="scan-1",
config=load_config(None),
trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
processors=[processor],
contract_mode="warn",
)
runtime.emit("stage_start", "rank", {})
assert "undeclared writes" in runtime.state["degradations"][0]
assert '"status": "degraded"' in (tmp_path / "events.jsonl").read_text()
def test_harness_config_serializes_models_processors_and_prompt_hashes(tmp_path: Path) -> None:
config_path = tmp_path / "openultrasast.toml" config_path = tmp_path / "openultrasast.toml"
config_path.write_text('[models]\nranker = "openrouter/ranker"\n') config_path.write_text('[models]\nranker = "openrouter/ranker"\n')
processor = RecordingProcessor(ProcessorSpec(name="recorder", version="2026-01-01", handles=("stage_start",), writes=("seen_stage",)))
output = tmp_path / "harness.json" output = tmp_path / "harness.json"
write_harness_config( write_harness_config(
config=load_config(config_path), config=load_config(config_path),
processors=[processor],
contract_mode="strict",
path=output, path=output,
prompts={"ranker": "score files"}, prompts={"ranker": "score files"},
) )
payload = json.loads(output.read_text()) payload = json.loads(output.read_text())
assert payload["model_roles"]["ranker"] == "openrouter/ranker" assert payload["model_roles"]["ranker"] == "openrouter/ranker"
assert payload["processor_versions"] == {"recorder": "2026-01-01"}
assert len(payload["prompt_hashes"]["ranker"]) == 64 assert len(payload["prompt_hashes"]["ranker"]) == 64
-198
View File
@@ -1,198 +0,0 @@
"""learning-harness task 1.5: curated repository tools and hunter-loop parameters (offline)."""
from __future__ import annotations
import json
import tempfile
from pathlib import Path
import pytest
from openultrasast.hunter_tools import PathEscapesRepo
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter
APP = (
"from flask import request\n\n\n"
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
" return Book.query.filter_by(book_title=title).first()\n\n\n"
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
" owner = request.args.get('owner')\n"
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n"
"def helper(value):\n"
" return value.strip()\n"
)
@pytest.fixture
def repo(tmp_path: Path) -> Path:
(tmp_path / "app.py").write_text(APP)
return tmp_path
def test_read_definition_returns_one_span_and_refuses_to_leave_the_repository(repo: Path) -> None:
from openultrasast.hunter_tools import read_definition
found = read_definition(repo, "leaky", max_chars=4000)
assert found is not None
assert found["path"] == "app.py" and found["name"] == "leaky"
assert found["start_line"] == 10 and found["end_line"] == 13
assert str(found["text"]).startswith("@app.route('/books/any/<title>')") # a handler's decorators come with it
assert "def leaky" in str(found["text"]) and "def guarded" not in str(found["text"])
assert read_definition(repo, "not_a_function", max_chars=4000) is None
with pytest.raises(PathEscapesRepo):
read_definition(repo, "leaky", max_chars=4000, path="../outside.py")
def test_entry_points_lists_handlers_with_their_access_and_never_source_text(repo: Path) -> None:
from openultrasast.hunter_tools import entry_points
rows = entry_points(repo)
by_function = {str(row["function"]): row for row in rows if row.get("function")}
assert {"guarded", "leaky"} <= set(by_function)
assert by_function["guarded"]["access_level"] == "authenticated" and by_function["leaky"]["access_level"] == "public"
assert by_function["leaky"]["path"] == "app.py" and isinstance(by_function["leaky"]["line"], int)
assert all("text" not in row and "content" not in row for row in rows) # identifiers and spans only
assert entry_points(repo, path="app.py") == rows
with pytest.raises(PathEscapesRepo):
entry_points(repo, path="../elsewhere.py")
def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path) -> None:
from openultrasast.hunter_tools import obligations
rows = obligations(repo, path="app.py", function="leaky")
assert rows and all(row["function"] == "leaky" for row in rows)
assert {"protected_read"} == {str(row["operation"]) for row in rows}
assert "identity_constraint" in {str(row["missing"]) for row in rows}
assert all("text" not in row and "content" not in row for row in rows)
assert obligations(repo, path="app.py", function="helper") == []
with pytest.raises(PathEscapesRepo):
obligations(repo, path="../outside.py", function="leaky")
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
from openultrasast.hunter_tools import flows
(repo / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n")
rows = flows(repo, path="flow.py", function="run")
assert all(row["function"] == "run" and row["path"] == "flow.py" for row in rows)
assert all(set(row) <= {"path", "function", "line", "sources", "sinks", "disposition", "cwe"} for row in rows)
assert flows(repo, path="app.py", function="helper") == []
with pytest.raises(PathEscapesRepo):
flows(repo, path="/etc/passwd", function="run")
def test_the_loop_keeps_todays_behaviour_by_default(repo: Path) -> None:
from openultrasast.complexity.map import Hotspot
from openultrasast.tool_hunter import DEFAULT_TOOL_NAMES
hotspot = Hotspot(
path="app.py",
function_name=None,
score=1.0,
band="high",
signals={},
rationale="fixture",
test_hint=None,
inventory_finding_ids=(),
)
client = ScriptedChatClient(
[
ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)),
ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'),
]
)
findings = run_tool_hunter(repo, [hotspot], client=client, model="m", max_steps=4)
assert [finding.finding_id for finding in findings] == ["tool-hunter:app.py:12"]
assert findings[0].tags == ["tool-hunter"] and findings[0].evidence_level == "suspicion"
assert tuple(tool["function"]["name"] for tool in client.calls[0]["tools"]) == DEFAULT_TOOL_NAMES
def test_the_loop_takes_a_prompt_context_tools_and_tags_and_replays_reasoning(repo: Path) -> None:
from openultrasast.complexity.map import Hotspot
hotspot = Hotspot(
path="app.py",
function_name="leaky",
score=1.0,
band="high",
signals={},
rationale="fixture",
test_hint=None,
inventory_finding_ids=(),
)
client = ScriptedChatClient(
[
ChatResponse(
content="looking",
reasoning="first I check the route",
tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),),
),
ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'),
]
)
findings = run_tool_hunter(
repo,
[hotspot],
client=client,
model="m",
max_steps=4,
system_prompt="You hunt access control only.",
user_prompt="Look at leaky.",
context_files=("app.py",),
tools=("read_definition", "obligations"),
tags=("family:access_control", "detector:access_control@1"),
)
first = client.calls[0]
assert first["messages"][0]["content"] == "You hunt access control only."
assert "Look at leaky." in str(first["messages"][1]["content"]) and "def leaky" in str(first["messages"][1]["content"])
assert [tool["function"]["name"] for tool in first["tools"]] == ["read_definition", "obligations"]
replayed = [message for message in client.calls[1]["messages"] if message.get("role") == "assistant"]
assert replayed and replayed[0]["reasoning_content"] == "first I check the route"
tool_reply = json.loads(str(client.calls[1]["messages"][-1]["content"]))
assert tool_reply and tool_reply[0]["missing"] == "identity_constraint"
assert findings[0].tags == ["family:access_control", "detector:access_control@1"]
def test_the_loop_asks_once_more_for_the_answer_instead_of_dropping_prose() -> None:
"""Measured live: the model spends its step budget on tools and then answers in prose, and the whole hunt is
discarded (learning-harness Req 6.3).
Four steps, three of them tool calls, and the fourth a paragraph that names the sink, the argument and the
line — parsed as nothing. A detector that found the bug and reported zero is indistinguishable from one that
found nothing, and every family in round zero would have baselined at zero for this reason alone."""
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter
prose = "The function passes `command` straight to os.system at line 2, an OS command sink."
answer = json.dumps([{"path": "app.py", "line": 2, "title": "command injection", "rationale": "os.system(command)"}])
client = ScriptedChatClient(
[
ChatResponse(tool_calls=(ToolCall(id="c1", name="grep_repo", arguments={"pattern": "system"}),)),
ChatResponse(tool_calls=(ToolCall(id="c2", name="read_file", arguments={"path": "app.py"}),)),
ChatResponse(content=prose),
ChatResponse(content=answer), # the one extra turn, asked for explicitly
]
)
with tempfile.TemporaryDirectory() as scratch:
root = Path(scratch)
(root / "app.py").write_text("import os\ndef run(command):\n return os.system(command)\n")
findings = run_tool_hunter(root, [], client=client, model="m", max_steps=3)
assert [item.line for item in findings] == [2]
final = client.calls[-1]
# Asked for JSON in the prompt, not through the provider's JSON mode: with tools attached, DeepSeek's JSON mode
# returned `{"type": "json_object"}` and lost the answers (tests/test_tool_hunter.py pins the shapes).
assert not final.get("json_object") and final.get("tools") == [], "the last turn is plain text without tools"
text = "\n".join(str(message.get("content") or "") for message in final["messages"])
assert "JSON" in text and prose in text, "the prose it already wrote is what the final turn is asked to format"
def test_a_hunt_that_never_called_a_tool_still_reports_nothing() -> None:
"""The extra turn is for formatting an answer, never for inventing one."""
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, run_tool_hunter
client = ScriptedChatClient([ChatResponse(content="I think there is an injection here."), ChatResponse(content="[]")])
with tempfile.TemporaryDirectory() as scratch:
root = Path(scratch)
(root / "app.py").write_text("x = 1\n")
assert run_tool_hunter(root, [], client=client, model="m", max_steps=3) == []
assert len(client.calls) == 1, "no tool was called, so there is nothing to format"
+6 -36
View File
@@ -3,7 +3,6 @@ from pathlib import Path
import pytest import pytest
from openultrasast.hunter_tools import PathEscapesRepo, clamp_repo_path, find_refs, grep_repo, read_file from openultrasast.hunter_tools import PathEscapesRepo, clamp_repo_path, find_refs, grep_repo, read_file
from openultrasast.mapping import EntryPointRecord
SPLIT_SINK = """\ SPLIT_SINK = """\
term = request.args["q"] term = request.args["q"]
@@ -85,42 +84,13 @@ def test_read_file_truncates_and_rejects_symlink_escape(tmp_path: Path) -> None:
read_file(root, "link.py", max_chars=100) read_file(root, "link.py", max_chars=100)
def test_find_refs_uses_mapping_index_and_clamped_text_search(tmp_path: Path) -> None: def test_find_refs_searches_source_text(tmp_path: Path) -> None:
root = _repo_with_split_sink(tmp_path) root = _repo_with_split_sink(tmp_path)
mapping_index = [ refs = find_refs(root, "query")
{"path": "app.py", "name": "query", "line": 2}, assert [ref["line"] for ref in refs] == [2, 3]
EntryPointRecord( assert all(ref["path"] == "app.py" and ref["name"] == "query" and ref["source"] == "text" for ref in refs)
path="app.py", assert find_refs(root, "search") == []
line=3, assert find_refs(root, "") == []
end_line=3,
function_name="search",
name="search",
kind="route",
access_level="public",
trust_boundary="http_request",
access_evidence=[],
conditions=[],
provenance="test",
rationale="fixture",
),
]
query_refs = find_refs(root, "query", mapping_index)
assert any(ref["path"] == "app.py" and ref["name"] == "query" for ref in query_refs)
assert any(ref.get("source") == "mapping" for ref in query_refs)
assert any(ref.get("source") == "text" and "query" in str(ref.get("text", ref.get("name", ""))) for ref in query_refs)
search_refs = find_refs(root, "search", mapping_index)
assert any(ref["path"] == "app.py" and ref["name"] == "search" for ref in search_refs)
def test_find_refs_rejects_mapping_path_escape(tmp_path: Path) -> None:
root = _repo_with_split_sink(tmp_path)
with pytest.raises(PathEscapesRepo):
find_refs(root, "passwd", [{"path": "../etc/passwd", "name": "passwd"}])
with pytest.raises(PathEscapesRepo):
find_refs(root, "passwd", [{"path": "/etc/passwd", "name": "passwd"}])
def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> None: def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> None:
+1 -26
View File
@@ -5,17 +5,7 @@ from __future__ import annotations
import pytest import pytest
from learn_fixtures import corpus from learn_fixtures import corpus
from openultrasast.learn.folds import ( from openultrasast.learn.folds import Fold, FoldError, assert_disjoint, compile_split, folds_digest, outer_folds
INSUFFICIENT,
Fold,
FoldError,
assert_disjoint,
compile_split,
folds_digest,
leave_one_framework_out,
leave_one_source_out,
outer_folds,
)
def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None: def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None:
@@ -26,8 +16,6 @@ def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation
assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4 assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4
folds = [ folds = [
*outer_folds(examples, seed=1, exclude=split.groups), *outer_folds(examples, seed=1, exclude=split.groups),
*leave_one_source_out(examples, exclude=split.groups),
*leave_one_framework_out(examples, exclude=split.groups, min_groups=2),
] ]
assert_disjoint(split, folds) assert_disjoint(split, folds)
for fold in folds: for fold in folds:
@@ -52,19 +40,6 @@ def test_folds_are_deterministic_by_seed() -> None:
assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6)) assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6))
def test_leave_one_source_and_framework_out() -> None:
examples = corpus(groups=12)
sources = leave_one_source_out(examples)
assert {f.held_source for f in sources} == {"pairs", "population-v1", "population-v2"}
for fold in sources:
assert fold.eval_groups == {e.group for e in examples if e.source == fold.held_source}
frameworks = {f.held_framework: f for f in leave_one_framework_out(examples)}
assert set(frameworks) == {"flask", "django"}
assert all(f.status == INSUFFICIENT and not f.eval_groups for f in frameworks.values()) # 3 groups each < 10
enough = {f.held_framework: f for f in leave_one_framework_out(examples, min_groups=3)}
assert enough["flask"].status == "ok" and enough["flask"].eval_groups == {e.group for e in examples if "flask" in e.frameworks}
def test_a_shared_group_is_refused() -> None: def test_a_shared_group_is_refused() -> None:
examples = corpus(groups=8) examples = corpus(groups=8)
split = compile_split(examples, seed=0) split = compile_split(examples, seed=0)
+3 -18
View File
@@ -8,7 +8,7 @@ from dataclasses import replace
import pytest import pytest
from learn_fixtures import by_id, corpus, example, x_record from learn_fixtures import by_id, corpus, example, x_record
from openultrasast.learn.folds import Fold, compile_split, leave_one_framework_out, leave_one_source_out, outer_folds from openultrasast.learn.folds import Fold, compile_split, outer_folds
from openultrasast.learn.retrieve import ( from openultrasast.learn.retrieve import (
BoundaryViolation, BoundaryViolation,
Target, Target,
@@ -166,13 +166,11 @@ def _all_folds(examples: list) -> list:
split = compile_split(examples, seed=3) split = compile_split(examples, seed=3)
return [ return [
*outer_folds(examples, seed=3, exclude=split.groups), *outer_folds(examples, seed=3, exclude=split.groups),
*leave_one_source_out(examples, exclude=split.groups),
*[f for f in leave_one_framework_out(examples, exclude=split.groups, min_groups=2) if f.status == "ok"],
] ]
def test_boundary() -> None: def test_boundary() -> None:
"""For outer, leave-one-source-out and leave-one-framework-out folds, every retrieved example and every """For outer folds, every retrieved example and every
demonstration that reaches a prompt is eligible; ``seed_memory`` writes no example of the case's group.""" demonstration that reaches a prompt is eligible; ``seed_memory`` writes no example of the case's group."""
examples = corpus(groups=16) examples = corpus(groups=16)
index = by_id(examples) index = by_id(examples)
@@ -186,7 +184,7 @@ def test_boundary() -> None:
) )
] ]
folds = _all_folds(examples) folds = _all_folds(examples)
assert {f.kind for f in folds} == {"outer", "leave_one_source_out", "leave_one_framework_out"} assert {f.kind for f in folds} == {"outer"}
checked = 0 checked = 0
for fold in folds: for fold in folds:
for target_example in [e for e in examples if e.group in fold.eval_groups]: for target_example in [e for e in examples if e.group in fold.eval_groups]:
@@ -206,19 +204,6 @@ def test_boundary() -> None:
assert all(e.group != group for e in seed_memory(examples, group)) assert all(e.group != group for e in seed_memory(examples, group))
def test_a_demonstration_of_the_held_out_framework_is_swapped_for_its_alternate() -> None:
flask = example("demo-flask", group="d/flask", label=1, frameworks=("flask",))
alt = example("demo-plain", group="d/plain", label=1)
target_example = example("t", group="t/t", label=0, frameworks=("flask",))
target = Target.of(target_example)
fold = leave_one_framework_out([flask, target_example], min_groups=1)[0]
assert fold.held_framework == "flask" and target_example.group in fold.eval_groups
demos = [{"example_id": flask.id, "alternate": {"example_id": alt.id}}]
usable = demonstrations_for(demos, by_id([flask, alt]), target, fold)
assert [d["example_id"] for d in usable] == [alt.id]
assert demonstrations_for([{"example_id": flask.id, "alternate": None}], by_id([flask]), target, fold) == []
def test_assert_boundary_catches_an_ineligible_rendered_id() -> None: def test_assert_boundary_catches_an_ineligible_rendered_id() -> None:
examples = corpus(groups=6) examples = corpus(groups=6)
fold = outer_folds(examples, k=3)[0] fold = outer_folds(examples, k=3)[0]
+1 -17
View File
@@ -7,7 +7,7 @@ from __future__ import annotations
import json import json
from pathlib import Path from pathlib import Path
from openultrasast.learn.features import Vocabulary, build_for_scan, function_of, model_sinks_part, roles_part from openultrasast.learn.features import Vocabulary, function_of, model_sinks_part, roles_part
from openultrasast.learn.roles import ( from openultrasast.learn.roles import (
RoleSet, RoleSet,
base_roles, base_roles,
@@ -168,22 +168,6 @@ def test_a_language_without_declarations_is_none_not_failed() -> None:
assert source_part(["def g():", " pass"], "python", "missing").state == "failed" assert source_part(["def g():", " pass"], "python", "missing").state == "failed"
def test_build_for_scan_records_the_roles_instrument(tmp_path: Path) -> None:
from openultrasast.findings import quick_scan_findings
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
_write(tmp_path, "app/one.py", "import os\n\n\ndef inner(value):\n os.system(value)\n")
_, targets = preprocess_repository(tmp_path)
assert targets and (tmp_path / targets[0].path).stat().st_size > 0, "the fixture was read"
findings = quick_scan_findings(tmp_path, targets, rank_targets(targets))
records = build_for_scan(tmp_path, findings, None)
assert records and records[0]["instruments"]["roles"]["state"] == "ran"
assert records[0]["x"]["roles.sink_confidence"] == 1.0
off = build_for_scan(tmp_path, findings, None, roles=False)
assert off[0]["instruments"]["roles"]["state"] == "none" and off[0]["x"]["roles.sink_confidence"] is None
# --- model roles: the plane task under a scripted client ---------------------------------------------------------- # --- model roles: the plane task under a scripted client ----------------------------------------------------------
+1 -57
View File
@@ -10,14 +10,10 @@ from typing import Any
import pytest import pytest
from openultrasast.findings import quick_scan_findings
from openultrasast.learn import schema from openultrasast.learn import schema
from openultrasast.learn.features import Delta, Part, build_for_scan, record from openultrasast.learn.features import Part, record
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
from openultrasast.plane.tasks.features import case_features from openultrasast.plane.tasks.features import case_features
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
APP = """import os APP = """import os
@@ -32,22 +28,6 @@ def {outer}():
""" """
def _scan(root: Path) -> list[dict[str, Any]]:
_, targets = preprocess_repository(root)
assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read"
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
findings = quick_scan_findings(root, targets, rank_targets(targets))
assert findings, "the fixture's os.system fires a quick rule"
engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []}
return build_for_scan(root, findings, engine)
def _repo(root: Path, path: str, inner: str, outer: str) -> Path:
(root / path).parent.mkdir(parents=True, exist_ok=True)
(root / path).write_text(APP.format(inner=inner, outer=outer))
return root
# --- the allow-list ---------------------------------------------------------------------------------------------- # --- the allow-list ----------------------------------------------------------------------------------------------
@@ -63,35 +43,10 @@ def test_features_are_allow_listed(tmp_path: Path) -> None:
mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"] mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary" assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS) assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta"))
assert records
for rec in records:
assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES"
assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"])
assert {s.name for s in schema.features_for("static")} < set(names) assert {s.name for s in schema.features_for("static")} < set(names)
assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"] assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]
def test_record_is_name_invariant(tmp_path: Path) -> None:
"""Two repositories that differ only in repository name, paths, file and function names and commit ids give the
same features: nothing identifies a repository."""
first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta"))
second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta"))
assert [r["x"] for r in first] == [r["x"] for r in second]
assert [r["instruments"] for r in first] == [r["instruments"] for r in second]
assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope"
assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None
root = tmp_path / "acme-shop"
_, targets = preprocess_repository(root)
findings = quick_scan_findings(root, targets, rank_targets(targets))
changed = {"app/one.py": [(5, 5)]}
one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40))
two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40))
assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40
assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1
assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature"
# --- label fields ------------------------------------------------------------------------------------------------ # --- label fields ------------------------------------------------------------------------------------------------
AGREED = { AGREED = {
@@ -165,17 +120,6 @@ def test_missing_is_not_zero() -> None:
assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero" assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"
def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None:
root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta")
_, targets = preprocess_repository(root)
findings = quick_scan_findings(root, targets, rank_targets(targets))
unread = build_for_scan(root, findings, {"questions": 0, "findings": []})
absent = build_for_scan(root, findings, None)
assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"}
assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"}
assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent))
def test_validate_record_refuses_values_outside_their_type() -> None: def test_validate_record_refuses_values_outside_their_type() -> None:
good = _plane()[0] good = _plane()[0]
validate_record(good) validate_record(good)
-56
View File
@@ -4,13 +4,7 @@ from pathlib import Path
from openultrasast.mapping import ( from openultrasast.mapping import (
analyze_entry_points, analyze_entry_points,
attach_reachability_hints, attach_reachability_hints,
attach_static_hints,
codeql_mapping_tasks,
differential_mapping_record,
ingest_sarif, ingest_sarif,
semgrep_mapping_tasks,
sharp_edge_record,
verifier_evidence_candidates,
write_entry_points, write_entry_points,
write_static_hints, write_static_hints,
) )
@@ -32,56 +26,6 @@ def test_ingest_sarif_normalizes_semgrep_hints(tmp_path: Path) -> None:
assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval" assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval"
def test_static_hints_attach_to_file_targets_and_evidence_candidates(tmp_path: Path) -> None:
repo = tmp_path / "repo"
repo.mkdir()
(repo / "app.py").write_text("eval(user_input)\n")
hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "py/code-injection", "app.py"))
_, targets = preprocess_repository(repo, static_hints=hints)
enriched = attach_static_hints(targets, hints)
evidence = verifier_evidence_candidates(hints, path="app.py")
assert enriched[0].static_hints[0]["analyzer"] == "codeql"
assert evidence == [
{
"source": "codeql",
"rule_id": "py/code-injection",
"path": "app.py",
"line": 7,
"severity": "high",
"message": "Potential injection",
"provenance": "sarif:codeql:py/code-injection",
}
]
def test_mapping_task_records_cover_disciplines(tmp_path: Path) -> None:
semgrep_hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "parser.c"))
codeql_hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "cpp/overflow", "parser.c"))
assert semgrep_mapping_tasks(semgrep_hints)[0].task == "pattern_variant_review"
assert codeql_mapping_tasks(codeql_hints)[0].task == "source_sink_sanitizer_path_review"
assert (
differential_mapping_record(
"parser.c",
change_kind="modified_function",
trust_boundary="network_input",
rationale="Parser changed behind a socket boundary.",
).trust_boundary
== "network_input"
)
assert (
sharp_edge_record(
"config.py",
category="insecure_default",
surface="debug mode",
rationale="Debug default changes production risk.",
).category
== "insecure_default"
)
def test_write_static_hints_artifact(tmp_path: Path) -> None: def test_write_static_hints_artifact(tmp_path: Path) -> None:
hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py")) hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py"))
output = tmp_path / "mapping" / "static_hints.json" output = tmp_path / "mapping" / "static_hints.json"
+24 -24
View File
@@ -26,17 +26,17 @@ def _cpg(rows): # type: ignore[no-untyped-def]
def test_a_permissive_literal_is_entailed() -> None: def test_a_permissive_literal_is_entailed() -> None:
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}] rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}]
answer = verdict(_cpg(rows), _spec(), function="create_app") answer = verdicts(_cpg(rows), _spec(), function="create_app")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
assert "*" in answer.witness and "CORS" in answer.witness assert "*" in answer.witness and "CORS" in answer.witness
def test_a_restrictive_literal_yields_no_finding() -> None: def test_a_restrictive_literal_yields_no_finding() -> None:
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
rows = [ rows = [
{ {
@@ -47,12 +47,12 @@ def test_a_restrictive_literal_yields_no_finding() -> None:
"args": ["app", '"https://app.example"'], "args": ["app", '"https://app.example"'],
} }
] ]
assert verdict(_cpg(rows), _spec(), function="create_app") is None assert verdicts(_cpg(rows), _spec(), function="create_app") == []
def test_a_non_literal_argument_is_corroborated_not_entailed() -> None: def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
"""The value is computed, so constant abstraction cannot decide it — that residual is the LLM's.""" """The value is computed, so constant abstraction cannot decide it — that residual is the LLM's."""
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
rows = [ rows = [
@@ -64,30 +64,30 @@ def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
"args": ["app", "configured_origins"], "args": ["app", "configured_origins"],
} }
] ]
answer = verdict(_cpg(rows), _spec(), function="create_app") answer = verdicts(_cpg(rows), _spec(), function="create_app")[0]
assert answer is not None and answer.rung is Rung.CORROBORATED assert answer is not None and answer.rung is Rung.CORROBORATED
def test_a_setting_that_is_not_in_the_spec_is_ignored() -> None: def test_a_setting_that_is_not_in_the_spec_is_ignored() -> None:
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
rows = [{"setting": 'print("*")', "line": "1", "method": "create_app", "literalArgs": ['"*"'], "args": ['"*"']}] rows = [{"setting": 'print("*")', "line": "1", "method": "create_app", "literalArgs": ['"*"'], "args": ['"*"']}]
assert verdict(_cpg(rows), _spec(), function="create_app") is None assert verdicts(_cpg(rows), _spec(), function="create_app") == []
def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None: def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None:
from openultrasast.cpg.backend import CpgResult from openultrasast.cpg.backend import CpgResult
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
assert verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") is None assert verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") == []
def test_the_verdict_is_deterministic() -> None: def test_the_verdict_is_deterministic() -> None:
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}] rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}]
cpg = _cpg(rows) cpg = _cpg(rows)
assert verdict(cpg, _spec(), function="login") == verdict(cpg, _spec(), function="login") assert verdicts(cpg, _spec(), function="login")[0] == verdicts(cpg, _spec(), function="login")[0]
# --- framework coverage and the weak-algorithm arm -------------------------------------------------------- # --- framework coverage and the weak-algorithm arm --------------------------------------------------------
@@ -113,24 +113,24 @@ def test_the_settings_table_covers_the_frameworks_in_the_corpus() -> None:
def test_a_weak_algorithm_is_entailed_by_its_literal() -> None: def test_a_weak_algorithm_is_entailed_by_its_literal() -> None:
"""CWE-327: the danger is the algorithm NAMED, not a permissive flag.""" """CWE-327: the danger is the algorithm NAMED, not a permissive flag."""
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.specs import config_specs from openultrasast.model.specs import config_specs
spec = config_specs(language="python")["config_secrets"] spec = config_specs(language="python")["config_secrets"]
rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}] rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}]
answer = verdict(_cpg(rows), spec, function="digest") answer = verdicts(_cpg(rows), spec, function="digest")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
assert "md5" in answer.witness.lower() assert "md5" in answer.witness.lower()
def test_a_strong_algorithm_yields_no_finding() -> None: def test_a_strong_algorithm_yields_no_finding() -> None:
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
from openultrasast.model.specs import config_specs from openultrasast.model.specs import config_specs
spec = config_specs(language="python")["config_secrets"] spec = config_specs(language="python")["config_secrets"]
rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}] rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}]
assert verdict(_cpg(rows), spec, function="digest") is None assert verdicts(_cpg(rows), spec, function="digest") == []
def _javascript_spec(): # type: ignore[no-untyped-def] def _javascript_spec(): # type: ignore[no-untyped-def]
@@ -142,7 +142,7 @@ def _javascript_spec(): # type: ignore[no-untyped-def]
def test_disabling_template_escaping_is_entailed() -> None: def test_disabling_template_escaping_is_entailed() -> None:
"""The setting NodeGoat's cross-site scripting turns off, claimed where the evidence is: the value.""" """The setting NodeGoat's cross-site scripting turns off, claimed where the evidence is: the value."""
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
rows = [ rows = [
@@ -154,7 +154,7 @@ def test_disabling_template_escaping_is_entailed() -> None:
"args": ["{ autoescape: false }"], "args": ["{ autoescape: false }"],
} }
] ]
answer = verdict(_cpg(rows), _javascript_spec(), function="<module>") answer = verdicts(_cpg(rows), _javascript_spec(), function="<module>")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
assert "swig.setDefaults" in answer.witness and "false" in answer.witness assert "swig.setDefaults" in answer.witness and "false" in answer.witness
@@ -167,7 +167,7 @@ def test_enabling_template_escaping_is_not_a_finding() -> None:
reported on exactly the code that repairs it -- the commented-out `autoescape: true` two lines below reported on exactly the code that repairs it -- the commented-out `autoescape: true` two lines below
NodeGoat's defect. NodeGoat's defect.
""" """
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
rows = [ rows = [
{ {
@@ -178,12 +178,12 @@ def test_enabling_template_escaping_is_not_a_finding() -> None:
"args": ["{ autoescape: true }"], "args": ["{ autoescape: true }"],
} }
] ]
assert verdict(_cpg(rows), _javascript_spec(), function="<module>") is None assert verdicts(_cpg(rows), _javascript_spec(), function="<module>") == []
def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None: def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
"""The control for the other direction: scoping must not quiet the settings it was not about.""" """The control for the other direction: scoping must not quiet the settings it was not about."""
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
rows = [ rows = [
@@ -195,7 +195,7 @@ def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
"args": ["{ origin: true }"], "args": ["{ origin: true }"],
} }
] ]
answer = verdict(_cpg(rows), _javascript_spec(), function="createApp") answer = verdicts(_cpg(rows), _javascript_spec(), function="createApp")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
@@ -206,13 +206,13 @@ def test_the_first_setting_in_a_file_is_the_one_reported() -> None:
spelling of its line numbers. NodeGoat has a permissive session at line 78 and template escaping disabled spelling of its line numbers. NodeGoat has a permissive session at line 78 and template escaping disabled
at line 135, and which one surfaced flipped on that comparison. at line 135, and which one surfaced flipped on that comparison.
""" """
from openultrasast.model.config_value import verdict from openultrasast.model.config_value import verdicts
rows = [ rows = [
{"setting": "swig.setDefaults({ autoescape: false })", "line": "135", "method": "m", "literalArgs": ["false"], "args": []}, {"setting": "swig.setDefaults({ autoescape: false })", "line": "135", "method": "m", "literalArgs": ["false"], "args": []},
{"setting": "cors({ origin: true })", "line": "78", "method": "m", "literalArgs": ["true"], "args": []}, {"setting": "cors({ origin: true })", "line": "78", "method": "m", "literalArgs": ["true"], "args": []},
] ]
answer = verdict(_cpg(rows), _javascript_spec(), function="m") answer = verdicts(_cpg(rows), _javascript_spec(), function="m")[0]
assert answer is not None and "line 78" in answer.witness, answer.witness assert answer is not None and "line 78" in answer.witness, answer.witness
-90
View File
@@ -1,90 +0,0 @@
"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1).
Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a
real error in this project's history:
per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the
hand-written development slice scored 83%
the overfitting gap published beside the headline, so the distance between the development slice and the
real-world one is never something a reader has to compute
the rung on every a finding's strength is what established it; a report that omits the rung invites the
finding reader to treat a suspicion as a result
"""
from __future__ import annotations
import pytest
def _slice(name, pairs, covered): # type: ignore[no-untyped-def]
from openultrasast.model.report import SliceRow
return SliceRow(slice=name, pairs=pairs, covered=covered)
def test_the_report_carries_a_row_per_slice() -> None:
from openultrasast.model.report import render
text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)])
assert "vibe-py" in text and "vfc-js" in text
assert "30" in text and "17" in text
def test_a_bare_aggregate_is_refused() -> None:
"""Req 11.1: no cross-slice number without the rows it was computed from."""
from openultrasast.model.report import render
with pytest.raises(ValueError, match="per-slice"):
render([])
def test_the_overfitting_gap_is_published_beside_the_headline() -> None:
"""The development slice minus the deciding real-world slice, stated rather than left to be computed."""
from openultrasast.model.report import overfitting_gap, render
rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)]
gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js")
assert gap == pytest.approx(0.80 - 0.20)
assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower()
assert "+60" in render(rows, development="vibe-py", deciding="vfc-js")
def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None:
"""A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one."""
from openultrasast.model.report import overfitting_gap
assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None
def test_every_finding_shows_its_rung() -> None:
from openultrasast.findings import StaticFinding
from openultrasast.model.ladder import Rung
from openultrasast.model.report import render_findings
finding = StaticFinding(
finding_id="r:app.py:1",
path="app.py",
title="t",
severity="high",
confidence="medium",
evidence_level="static_corroboration",
rationale="why",
line=1,
function_name="run",
reachability_status="unknown",
reachability_evidence=[],
reachability_conditions=[],
tags=[],
ranking_priority=1.0,
rung=Rung.ENTAILED,
)
text = render_findings([finding])
assert "model_entailed" in text and "app.py" in text
def test_the_deciding_slice_is_named_in_the_report() -> None:
"""Req 9.3: the real-world slice decides, and the report says which one that is."""
from openultrasast.model.report import render
text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js")
assert "vfc-js" in text and "decid" in text.lower()
+43 -45
View File
@@ -30,12 +30,12 @@ def _spec(): # type: ignore[no-untyped-def]
def test_an_unsanitized_source_to_sink_path_is_entailed() -> None: def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3} {"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3}
] ]
answer = verdict(_cpg(rows), _spec(), function="run") answer = verdicts(_cpg(rows), _spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
assert answer.family == "injection" assert answer.family == "injection"
assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found" assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found"
@@ -44,30 +44,30 @@ def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None: def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None:
"""The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's.""" """The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's."""
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}] rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}]
answer = verdict(_cpg(rows), _spec(), function="run") answer = verdicts(_cpg(rows), _spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.CORROBORATED assert answer is not None and answer.rung is Rung.CORROBORATED
def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None: def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None:
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
assert verdict(_cpg([]), _spec(), function="run") is None assert verdicts(_cpg([]), _spec(), function="run") == []
def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None: def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None:
"""`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3.""" """`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3."""
from openultrasast.cpg.backend import CpgResult from openultrasast.cpg.backend import CpgResult
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None) broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None)
assert verdict(broken, _spec(), function="run") is None assert verdicts(broken, _spec(), function="run") == []
def test_the_verdict_is_restricted_to_the_labeled_function() -> None: def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -79,20 +79,20 @@ def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
"length": 2, "length": 2,
} }
] ]
assert verdict(_cpg(rows), _spec(), function="run") is None, "a flow in another function does not judge this one" assert verdicts(_cpg(rows), _spec(), function="run") == [], "a flow in another function does not judge this one"
def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None: def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None:
"""Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average.""" """Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average."""
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9}, {"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9},
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2}, {"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2},
] ]
first = verdict(_cpg(rows), _spec(), function="run") first = verdicts(_cpg(rows), _spec(), function="run")[0]
second = verdict(_cpg(rows), _spec(), function="run") second = verdicts(_cpg(rows), _spec(), function="run")[0]
assert first == second assert first == second
assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness
@@ -100,7 +100,7 @@ def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic()
def test_the_spec_drives_the_query_parameters() -> None: def test_the_spec_drives_the_query_parameters() -> None:
"""The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam.""" """The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam."""
from openultrasast.cpg.backend import CpgResult from openultrasast.cpg.backend import CpgResult
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
seen: dict[str, object] = {} seen: dict[str, object] = {}
@@ -109,7 +109,7 @@ def test_the_spec_drives_the_query_parameters() -> None:
seen.update(params) seen.update(params)
return [] return []
verdict(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run") verdicts(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run")
assert seen["query"] == "taint" assert seen["query"] == "taint"
assert seen["sources"] == ("request.args",) assert seen["sources"] == ("request.args",)
assert seen["sinks"] == ("os.system", "execute") assert seen["sinks"] == ("os.system", "execute")
@@ -143,7 +143,7 @@ def _shape_spec(): # type: ignore[no-untyped-def]
def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_reaches_it() -> None: def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_reaches_it() -> None:
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -157,12 +157,12 @@ def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_r
"sinkArg0Literal": True, "sinkArg0Literal": True,
} }
] ]
assert verdict(_cpg(rows), _shape_spec(), function="run") is None, "a bound query is the fix, not the bug" assert verdicts(_cpg(rows), _shape_spec(), function="run") == [], "a bound query is the fix, not the bug"
def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None: def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -176,14 +176,14 @@ def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
"sinkArg0Literal": False, "sinkArg0Literal": False,
} }
] ]
answer = verdict(_cpg(rows), _shape_spec(), function="run") answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> None: def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> None:
"""`os.system(x)` with one argument is not "safe because it has one argument" -- the rule is per sink.""" """`os.system(x)` with one argument is not "safe because it has one argument" -- the rule is per sink."""
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -197,17 +197,17 @@ def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> N
"sinkArg0Literal": False, "sinkArg0Literal": False,
} }
] ]
answer = verdict(_cpg(rows), _spec(), function="run") # os.system is not in safe_shape_sinks answer = verdicts(_cpg(rows), _spec(), function="run")[0] # os.system is not in safe_shape_sinks
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
def test_a_row_without_shape_fields_is_judged_on_the_flow_alone() -> None: def test_a_row_without_shape_fields_is_judged_on_the_flow_alone() -> None:
"""Older rows, or an engine that could not report arity, must not be silently treated as safe.""" """Older rows, or an engine that could not report arity, must not be silently treated as safe."""
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [{"sink": "db.execute(q)", "sinkLine": "5", "sinkMethod": "run", "source": "request.args['n']", "sanitized": False, "length": 3}] rows = [{"sink": "db.execute(q)", "sinkLine": "5", "sinkMethod": "run", "source": "request.args['n']", "sanitized": False, "length": 3}]
answer = verdict(_cpg(rows), _shape_spec(), function="run") answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
@@ -219,7 +219,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
So the flag must exist and must default to off. So the flag must exist and must default to off.
""" """
from openultrasast.cpg.backend import CpgResult from openultrasast.cpg.backend import CpgResult
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
seen: dict[str, object] = {} seen: dict[str, object] = {}
@@ -227,9 +227,9 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
seen.update(params) seen.update(params)
return [] return []
verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run") verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run")
assert seen["parameterSources"] == "false", "treating every parameter as untrusted is not the default" assert seen["parameterSources"] == "false", "treating every parameter as untrusted is not the default"
verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True) verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True)
assert seen["parameterSources"] == "true" assert seen["parameterSources"] == "true"
@@ -250,7 +250,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None: def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None:
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -263,13 +263,13 @@ def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None
"inLabeledScope": True, "inLabeledScope": True,
} }
] ]
answer = verdict(_cpg(rows), _spec(), function="requestListener") answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
assert "<lambda>0" not in answer.witness, "the witness names the flow, not the synthetic closure name" assert "<lambda>0" not in answer.witness, "the witness names the flow, not the synthetic closure name"
def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None: def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -282,17 +282,17 @@ def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
"inLabeledScope": False, "inLabeledScope": False,
} }
] ]
assert verdict(_cpg(rows), _spec(), function="requestListener") is None assert verdicts(_cpg(rows), _spec(), function="requestListener") == []
def test_rows_without_the_scope_flag_fall_back_to_the_method_name() -> None: def test_rows_without_the_scope_flag_fall_back_to_the_method_name() -> None:
"""A row from an engine that cannot report scope must not silently become in-scope.""" """A row from an engine that cannot report scope must not silently become in-scope."""
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{"sink": "os.system(x)", "sinkLine": "9", "sinkMethod": "<lambda>0", "source": "request.args['x']", "sanitized": False, "length": 2} {"sink": "os.system(x)", "sinkLine": "9", "sinkMethod": "<lambda>0", "source": "request.args['x']", "sanitized": False, "length": 2}
] ]
assert verdict(_cpg(rows), _spec(), function="run") is None assert verdicts(_cpg(rows), _spec(), function="run") == []
def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter() -> None: def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter() -> None:
@@ -305,7 +305,7 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter(
breaks ties, so the order stays total and the verdict deterministic. breaks ties, so the order stays total and the verdict deterministic.
""" """
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -336,10 +336,10 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter(
"inLabeledScope": True, "inLabeledScope": True,
}, },
] ]
answer = verdict(_cpg(rows), _spec(), function="requestListener") answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
assert "request.args['f']" in answer.witness, f"witness named no real source: {answer.witness}" assert "request.args['f']" in answer.witness, f"witness named no real source: {answer.witness}"
assert verdict(_cpg(rows), _spec(), function="requestListener") == answer # still deterministic assert verdicts(_cpg(rows), _spec(), function="requestListener")[0] == answer # still deterministic
def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitized() -> None: def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitized() -> None:
@@ -351,7 +351,7 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize
modelled source reaches the sink, those flows are the evidence and the bare-parameter ones are ignored. modelled source reaches the sink, those flows are the evidence and the bare-parameter ones are ignored.
""" """
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -373,14 +373,14 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize
"inLabeledScope": True, "inLabeledScope": True,
}, },
] ]
answer = verdict(_cpg(rows), _spec(), function="requestListener") answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
assert answer is not None and answer.rung is Rung.CORROBORATED, "a sanitized real flow is not an entailment" assert answer is not None and answer.rung is Rung.CORROBORATED, "a sanitized real flow is not an entailment"
assert "request.args['f']" in answer.witness assert "request.args['f']" in answer.witness
def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() -> None: def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() -> None:
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
rows = [ rows = [
{ {
@@ -393,7 +393,7 @@ def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink()
"inLabeledScope": True, "inLabeledScope": True,
} }
] ]
answer = verdict(_cpg(rows), _spec(), function="run") answer = verdicts(_cpg(rows), _spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED assert answer is not None and answer.rung is Rung.ENTAILED
@@ -443,13 +443,13 @@ def test_a_bound_that_governs_the_sink_lowers_the_rung() -> None:
from openultrasast.cpg.backend import CpgResult from openultrasast.cpg.backend import CpgResult
from openultrasast.model.ladder import Rung from openultrasast.model.ladder import Rung
from openultrasast.model.specs import taint_specs from openultrasast.model.specs import taint_specs
from openultrasast.model.taint import verdict from openultrasast.model.taint import verdicts
spec = taint_specs(language="c")["memory"] spec = taint_specs(language="c")["memory"]
assert "strcpy" in spec.bounded_sinks, "a CWE-121 sink is dischargeable by a bound" assert "strcpy" in spec.bounded_sinks, "a CWE-121 sink is dischargeable by a bound"
governed = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main") governed = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main")[0]
ungoverned = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main") ungoverned = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main")[0]
assert governed is not None and governed.rung is Rung.CORROBORATED assert governed is not None and governed.rung is Rung.CORROBORATED
assert "governed by" in governed.witness, "the witness names the bound it found" assert "governed by" in governed.witness, "the witness names the bound it found"
@@ -471,7 +471,7 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None:
verdict, and `system` won it on flow length, so the SQL injection went unreported by construction.""" verdict, and `system` won it on flow length, so the SQL injection went unreported by construction."""
from openultrasast.cpg.backend import CpgResult from openultrasast.cpg.backend import CpgResult
from openultrasast.model.specs import taint_specs from openultrasast.model.specs import taint_specs
from openultrasast.model.taint import verdict, verdicts from openultrasast.model.taint import verdicts
def row(sink, line, length): # type: ignore[no-untyped-def] def row(sink, line, length): # type: ignore[no-untyped-def]
return { return {
@@ -497,5 +497,3 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None:
assert len(every) == 2, "two sink sites are two findings" assert len(every) == 2, "two sink sites are two findings"
assert {v.location for v in every} == {"app.php:6:handler", "app.php:13:handler"} assert {v.location for v in every} == {"app.php:6:handler", "app.php:13:handler"}
# The pair path is unchanged: the strongest is still what `verdict` returns.
assert every[0].location == verdict(cpg, spec, function="", file="app.php").location
-16
View File
@@ -62,22 +62,6 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None
assert taxonomy.family_of_cwe("CWE-99999") is None assert taxonomy.family_of_cwe("CWE-99999") is None
def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None:
from openultrasast.model.taxonomy import load_families
taxonomy = load_families()
assert taxonomy.related("injection", "injection") == "same"
assert taxonomy.related("injection", "access_control") == "lateral"
assert taxonomy.related("injection", "not_a_family") == "fabricated"
assert taxonomy.related("unknown", "injection") == "lateral" # abstention earns no partial credit
assert all(family.parent is None for family in taxonomy.families) # the shipped taxonomy is flat
(tmp_path / "families.toml").write_text(
GOOD + '\n[[family]]\nid = "path"\nparent = "injection"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "canary"\n'
)
scoped = load_families(tmp_path / "families.toml", require_all=False)
assert scoped.related("path", "injection") == "parent_child" and scoped.related("injection", "path") == "parent_child"
@pytest.mark.parametrize( @pytest.mark.parametrize(
("text", "needle"), ("text", "needle"),
[ [
-10
View File
@@ -129,16 +129,6 @@ def test_state_is_mirrored_and_seeds_a_rerun_elsewhere(store: HttpObjectStore, t
assert second.load_state() is None assert second.load_state() is None
def test_put_input_stores_a_file_no_task_produces(store: HttpObjectStore, tmp_path: Path) -> None:
delivery = Delivery(store, "r", tmp_path / "r")
source = tmp_path / "candidates.json"
source.write_bytes(b"[1, 2, 3]")
url = delivery.put_input("candidates.json", source)
assert store.objects["runs/r/inputs/candidates.json"] == b"[1, 2, 3]"
with urllib.request.urlopen(url, timeout=10) as response: # noqa: S310
assert response.read() == b"[1, 2, 3]"
def test_declared_artifacts_are_outputs_plus_what_consumers_name(tmp_path: Path) -> None: def test_declared_artifacts_are_outputs_plus_what_consumers_name(tmp_path: Path) -> None:
"""A consumer may declare a producer's summary.json (every task writes one) without the producer listing it.""" """A consumer may declare a producer's summary.json (every task writes one) without the producer listing it."""
from openultrasast.plane.manifests import load_manifests from openultrasast.plane.manifests import load_manifests
+3 -58
View File
@@ -3,8 +3,7 @@
Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the memory store's Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the memory store's
host (where the runner PUTs its tar and GETs its inputs), the bound Workspaces' Git hosts and the bound Model's API host (where the runner PUTs its tar and GETs its inputs), the bound Workspaces' Git hosts and the bound Model's API
host, all as one TLS passthrough rule on 443 (plane-on-kubernetes 2.3), and nothing a gateway would reject or that host, all as one TLS passthrough rule on 443 (plane-on-kubernetes 2.3), and nothing a gateway would reject or that
would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule (the receiver path, kept for would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule.
group 2 only, is the one exception).
""" """
from __future__ import annotations from __future__ import annotations
@@ -12,12 +11,11 @@ from __future__ import annotations
import ipaddress import ipaddress
import json import json
import threading import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
import pytest import pytest
from openultrasast.plane import egress, runner from openultrasast.plane import egress
from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest
MANIFESTS = """ MANIFESTS = """
@@ -63,7 +61,7 @@ def hostnames(policy: dict) -> list[str]:
def assert_tight(policy: dict) -> None: def assert_tight(policy: dict) -> None:
for rule in policy["rules"]: for rule in policy["rules"]:
assert set(rule) <= {"http", "tlsPassthrough"}, f"only http and passthrough rules, got {rule}" assert set(rule) == {"tlsPassthrough"}, f"only TLS passthrough rules, got {rule}"
body = next(iter(rule.values())) body = next(iter(rule.values()))
assert "all" not in body["ports"], "never every port" assert "all" not in body["ports"], "never every port"
for host in hostnames(policy): for host in hostnames(policy):
@@ -82,12 +80,6 @@ def test_model_free_task_gets_only_the_store(manifests) -> None:
assert egress.policy_for(manifests.tasks["facts"], [], None, None) == {"rules": []}, "no store, no Git, no Model: nothing" assert egress.policy_for(manifests.tasks["facts"], [], None, None) == {"rules": []}, "no store, no Git, no Model: nothing"
def test_the_receiver_path_adds_the_one_http_rule_until_it_is_removed(manifests) -> None:
policy = egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host=egress.RECEIVER_HOST)
assert policy == {"rules": [{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}}]}
assert egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host="127.0.0.1") == {"rules": []}
def test_model_bound_task_with_git_workspaces(manifests) -> None: def test_model_bound_task_with_git_workspaces(manifests) -> None:
task = manifests.tasks["verify"] task = manifests.tasks["verify"]
policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], STORE) policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], STORE)
@@ -191,50 +183,3 @@ def test_apply_waits_for_the_actor_then_gives_up(tmp_path: Path, kubectl_ate: Pa
timer.start() timer.start()
cli.apply("late", "default", POLICY, timeout=10) cli.apply("late", "default", POLICY, timeout=10)
timer.join() timer.join()
def test_receiver_address_modes(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
assert egress.receiver_address(4242, None) == ("http://127.0.0.1:4242/", None)
monkeypatch.delenv("OUSAST_ARTIFACT_HOST")
monkeypatch.setenv("OUSAST_ARTIFACT_DIAL", "172.19.0.1:80")
assert egress.receiver_address(18090, "kind-test") == (f"http://{egress.RECEIVER_HOST}/", "172.19.0.1:80")
monkeypatch.delenv("OUSAST_ARTIFACT_DIAL")
with pytest.raises(RuntimeError, match="kube context"):
egress.receiver_address(18090, None) # no context to look the Service up in: never a built one
# --- the runner dials the gateway address with the receiver's name as Host ----------------------------------------
class _Seen(ThreadingHTTPServer):
daemon_threads = True
seen: list[tuple[str, str | None, bytes]]
class _Handler(BaseHTTPRequestHandler):
server: _Seen
def do_POST(self) -> None: # noqa: N802
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
self.server.seen.append((self.path, self.headers.get("Host"), body))
self.send_response(200)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
def test_delivery_dials_the_override_with_the_url_host() -> None:
server = _Seen(("127.0.0.1", 0), _Handler)
server.seen = []
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
dial = f"127.0.0.1:{server.server_address[1]}"
url = f"http://{egress.RECEIVER_HOST}/in"
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, dial=dial) == 200
assert server.seen == [("/in", egress.RECEIVER_HOST, b"tar")]
finally:
server.shutdown()
server.server_close()
+1 -34
View File
@@ -9,10 +9,9 @@ from pathlib import Path
import pytest import pytest
from openultrasast.learn.schema import FeatureRecordError, feature_set_digest
from openultrasast.plane.memory import FileStore from openultrasast.plane.memory import FileStore
from openultrasast.plane.tasks import features from openultrasast.plane.tasks import features
from openultrasast.plane.tasks.remember import Context, remember_run, rows_for from openultrasast.plane.tasks.remember import remember_run
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46" FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
CASE = "lmdeploy-media-url-ssrf" CASE = "lmdeploy-media-url-ssrf"
@@ -25,30 +24,6 @@ def run_dir(tmp_path: Path) -> Path:
return target return target
def _checkout(root: Path) -> Path:
media = root / "lmdeploy" / "vl" / "media"
media.mkdir(parents=True)
(media / "connection.py").write_text("def _is_safe_url(u):\n return True\n\n\ndef _load_http_url(u):\n x = 1\n return x\n")
return root
def test_features_of_the_recorded_case(run_dir: Path, tmp_path: Path) -> None:
[rec] = features.recorded_case(run_dir, CASE, workspace=_checkout(tmp_path / "checkout"))
assert (rec["candidate"], rec["family"], rec["profile"], rec["unit"]) == (
"lmdeploy/vl/media/connection.py::_load_http_url", "untrusted_destination", "plane", "pin",
) # fmt: skip
x, states = rec["x"], {k: v["state"] for k, v in rec["instruments"].items()}
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, True, None, 2)
assert x["agree.final"] == "agreed" and x["facts.callers"] == 0 and x["facts.is_global"] is False
assert x["facts.function_lines"] == 3 and x["lang.python"] is True and x["verify.site_offset"] == 0
assert states["verify"] == states["agree"] == states["facts"] == states["source"] == "ran"
assert states["quick"] == states["engine"] == "none", "no alerts task in this Run: no coverage, not zero hits"
assert rec["instruments"]["verify"]["version"] == "deepseek-flash" and rec["feature_set_digest"] == feature_set_digest()
assert "site_match" not in json.dumps(rec), "the fixture's site_match: true was dropped on read"
[without] = features.recorded_case(run_dir, CASE)
assert without["instruments"]["source"]["state"] == "none" and without["x"]["facts.function_lines"] is None
def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None: def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None:
out = tmp_path / "out" out = tmp_path / "out"
env = { env = {
@@ -71,13 +46,5 @@ def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_pat
assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed" assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed"
def test_remember_refuses_a_record_outside_the_allow_list(run_dir: Path) -> None:
[rec] = features.recorded_case(run_dir, CASE)
rec["x"]["site_match"] = True
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
with pytest.raises(FeatureRecordError, match="site_match"):
rows_for(ctx, facts=None, passes={}, models={}, agreed=None, features=[rec])
def test_the_task_fails_loudly_without_an_output_dir() -> None: def test_the_task_fails_loudly_without_an_output_dir() -> None:
assert features.main({}) == 2 assert features.main({}) == 2
+1 -1
View File
@@ -77,7 +77,7 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls
env = {e.name: e.value for e in task.env} env = {e.name: e.value for e in task.env}
assert env["OUSAST_PASS"] == entry.name[-1] assert env["OUSAST_PASS"] == entry.name[-1]
docs = reconciler.render_task(run, entry, manifests, "http://127.0.0.1:1/") docs = reconciler.render_task(run, entry, manifests)
for doc in docs: for doc in docs:
assert not non_ax_fields(doc["spec"], AX_FIELDS[doc["kind"]], doc["kind"]) assert not non_ax_fields(doc["spec"], AX_FIELDS[doc["kind"]], doc["kind"])
assert set(doc["metadata"]) <= {"name", "atespace"} assert set(doc["metadata"]) <= {"name", "atespace"}
+5 -92
View File
@@ -5,7 +5,7 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates
fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
-- and on the scripted poll it PUTs a scripted tar to the presigned ``delivery.put`` URL the start request carried -- and on the scripted poll it PUTs a scripted tar to the presigned ``delivery.put`` URL the start request carried
(plane-on-kubernetes 2.4; or POSTs it to ``OUSAST_ARTIFACT_URL`` in receiver mode), or turns ``Failed`` with a (plane-on-kubernetes 2.4), or turns ``Failed`` with a
condition message, or vanishes. Every outcome comes from a scripted delivery, and a delivery happens only after condition message, or vanishes. Every outcome comes from a scripted delivery, and a delivery happens only after
the reconciler's start request reached the fake router (Req 4.5). The store is ``plane_fake_store.HttpObjectStore``. the reconciler's start request reached the fake router (Req 4.5). The store is ``plane_fake_store.HttpObjectStore``.
""" """
@@ -21,8 +21,6 @@ import sys
import tarfile import tarfile
import threading import threading
import time import time
import urllib.error
import urllib.request
from collections.abc import Iterator from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
@@ -152,11 +150,9 @@ elif argv[:2] == ["get", "task"]:
delivery = body.get("delivery") or {{}} delivery = body.get("delivery") or {{}}
if script.get("deliver", True) and "phase" not in script and not script.get("vanish"): if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
files = script.get("files", {{}}) files = script.get("files", {{}})
base = env.get("OUSAST_ARTIFACT_URL", "").rstrip("/") + "/inputs/"
for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items(): for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items():
presigned = (delivery.get("inputs") or {{}}).get(name) presigned = (delivery.get("inputs") or {{}}).get(name)
fetch = urllib.request.Request(presigned) if presigned else urllib.request.Request( fetch = urllib.request.Request(presigned)
base + ref, headers={{"X-Ousast-Run": env["OUSAST_RUN"], "X-Ousast-Task": task}})
try: try:
with urllib.request.urlopen(fetch) as resp: with urllib.request.urlopen(fetch) as resp:
dest = HERE / "fetched" / task / ref dest = HERE / "fetched" / task / ref
@@ -175,13 +171,8 @@ elif argv[:2] == ["get", "task"]:
info = tarfile.TarInfo(path) info = tarfile.TarInfo(path)
info.size = len(data) info.size = len(data)
tar.addfile(info, io.BytesIO(data)) tar.addfile(info, io.BytesIO(data))
headers = {{"Content-Type": "application/x-tar", "X-Ousast-Run": env["OUSAST_RUN"]}}
headers["X-Ousast-Task"] = script.get("as_task", task)
if delivery.get("put"): # the store path: PUT to the presigned URL, no receiver headers
tar_headers = {{"Content-Type": "application/x-tar"}} tar_headers = {{"Content-Type": "application/x-tar"}}
req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers) req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers)
else:
req = urllib.request.Request(env["OUSAST_ARTIFACT_URL"], data=buf.getvalue(), method="POST", headers=headers)
try: try:
with urllib.request.urlopen(req) as resp: with urllib.request.urlopen(req) as resp:
(HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n") (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n")
@@ -420,7 +411,6 @@ def test_chain_runs_in_order_with_inputs_env_and_model(fake: Fake, tmp_path: Pat
assert env["OUSAST_BUDGET_USD"] == "2.5" and env["OUSAST_BUDGET_CALLS"] == "40" assert env["OUSAST_BUDGET_USD"] == "2.5" and env["OUSAST_BUDGET_CALLS"] == "40"
assert env["OUSAST_INPUT_FACTS"] == "/workspace/.ousast-in/verify/repo-facts/facts.json" assert env["OUSAST_INPUT_FACTS"] == "/workspace/.ousast-in/verify/repo-facts/facts.json"
assert json.loads(env["OUSAST_INPUTS"]) == {"FACTS": "repo-facts/facts.json"} assert json.loads(env["OUSAST_INPUTS"]) == {"FACTS": "repo-facts/facts.json"}
assert "OUSAST_ARTIFACT_URL" not in env, "the store path renders no receiver address into the Task"
assert env["OUSAST_MODEL"] == "deepseek-flash" assert env["OUSAST_MODEL"] == "deepseek-flash"
assert json.loads(env["OUSAST_MODEL_PARAMS"]) == {"cache_hit_per_m": 0.07, "input_per_m": 0.27, "output_per_m": 1.1} assert json.loads(env["OUSAST_MODEL_PARAMS"]) == {"cache_hit_per_m": 0.07, "input_per_m": 0.27, "output_per_m": 1.1}
kinds = [(d["kind"], d["metadata"]["name"]) for d in verify["docs"]] kinds = [(d["kind"], d["metadata"]["name"]) for d in verify["docs"]]
@@ -661,33 +651,6 @@ def tar_of(files: dict[str, bytes]) -> bytes:
return buf.getvalue() return buf.getvalue()
def test_receiver_rejects_traversal_unknown_tasks_and_wrong_types(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") # the direct test address: no Service lookup, no context needed
receiver = reconciler.Receiver("r", tmp_path / "r", 0)
receiver.running.add("t")
headers = {"X-Ousast-Run": "r", "X-Ousast-Task": "t", "Content-Type": "application/x-tar"}
code, text = receiver.accept(headers, tar_of({"../escape.json": b"{}"}))
assert code == 400 and "escape" in text and not (tmp_path / "escape.json").exists()
assert receiver.accept({**headers, "X-Ousast-Task": "other"}, tar_of({"summary.json": b"{}"}))[0] == 403
assert receiver.accept({**headers, "X-Ousast-Run": "someone"}, tar_of({"summary.json": b"{}"}))[0] == 403
assert receiver.accept({**headers, "Content-Type": "application/json"}, b"{}")[0] == 415
assert receiver.accept(headers, b"not a tar")[0] == 400
assert receiver.accept(headers, tar_of({"summary.json": b"{}", "sub/x.json": b"1"}))[0] == 200
assert (tmp_path / "r" / "t" / "sub" / "x.json").read_text() == "1" and "t" in receiver.delivered
receiver.server_close()
def test_delivery_for_a_task_that_is_not_running_is_rejected(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_DELIVERY", "receiver") # the receiver path, kept for group 2 (task 2.6 removes this test)
monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "1") # the rejected delivery is no delivery: the task times out
script = json.loads(json.dumps(SCRIPT))
script["repo-facts"]["as_task"] = "agree" # the runner claims to be a task that has not started
fake.script(script)
assert reconciler.run(write_run(tmp_path, "spoof"), ax=str(fake.ax)) == "failed"
assert [n for _, n in fake.events("rejected")] == ["repo-facts"]
assert not (reconciler.run_dir("spoof") / "agree" / "summary.json").exists()
def test_lock_refuses_a_second_runner_and_ignores_a_dead_one(fake: Fake, tmp_path: Path) -> None: def test_lock_refuses_a_second_runner_and_ignores_a_dead_one(fake: Fake, tmp_path: Path) -> None:
manifest = write_run(tmp_path, "locked") manifest = write_run(tmp_path, "locked")
base = reconciler.run_dir("locked") base = reconciler.run_dir("locked")
@@ -824,7 +787,7 @@ def test_rendered_documents_carry_only_ax_fields(tmp_path: Path) -> None:
(tmp_path / "repo-facts").mkdir() (tmp_path / "repo-facts").mkdir()
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8") (tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
run, manifests = reconciler.load_run(write_run(tmp_path, "fields")) run, manifests = reconciler.load_run(write_run(tmp_path, "fields"))
docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests, "http://h:1/")] docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests)]
assert {d["kind"] for d in docs} == {"Task", "Workspace", "Model"} assert {d["kind"] for d in docs} == {"Task", "Workspace", "Model"}
for rendered in docs: for rendered in docs:
where = f"{rendered['kind']}/{rendered['metadata']['name']}" where = f"{rendered['kind']}/{rendered['metadata']['name']}"
@@ -841,7 +804,7 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None:
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8") (tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
run, manifests = reconciler.load_run(write_run(tmp_path, "order")) run, manifests = reconciler.load_run(write_run(tmp_path, "order"))
for entry in run.tasks: for entry in run.tasks:
docs = reconciler.render_task(run, entry, manifests, "http://h:1/") docs = reconciler.render_task(run, entry, manifests)
assert [d["kind"] for d in docs].count("Task") == 1 assert [d["kind"] for d in docs].count("Task") == 1
assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs] assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs]
@@ -943,44 +906,6 @@ def test_a_refused_egress_policy_fails_the_task_before_resume(fake: Fake, tmp_pa
SUBSTRATE_ENV_LIMIT = 32768 # "actor_template.containers[0].env[11].value: Too long" on the live cluster, 2026-09-29 SUBSTRATE_ENV_LIMIT = 32768 # "actor_template.containers[0].env[11].value: Too long" on the live cluster, 2026-09-29
def get_input(receiver: reconciler.Receiver, path: str, run: str = "r", task: str = "t") -> tuple[int, bytes]:
url = f"http://127.0.0.1:{receiver.server_address[1]}{path}"
request = urllib.request.Request(url, headers={"X-Ousast-Run": run, "X-Ousast-Task": task})
try:
with urllib.request.urlopen(request, timeout=10) as response:
return response.status, response.read()
except urllib.error.HTTPError as exc:
return exc.code, exc.read()
def test_receiver_serves_a_running_task_its_declared_inputs_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
base = tmp_path / "r"
(base / "p").mkdir(parents=True)
big = b"x" * (3 << 20)
(base / "p" / "a.json").write_bytes(big)
(base / "p" / "b.json").write_bytes(b"not declared")
(tmp_path / "secret.txt").write_text("outside the run")
receiver = reconciler.Receiver("r", base, 0)
threading.Thread(target=receiver.serve_forever, daemon=True).start()
try:
receiver.running.update({"t", "idle"})
receiver.inputs.update({"t": frozenset({"p/a.json", "p/missing.json", "../secret.txt"}), "done": frozenset({"p/a.json"})})
assert get_input(receiver, "/inputs/p/a.json") == (200, big), "streamed whole"
assert get_input(receiver, "/inputs/p/a.json", run="other")[0] == 403, "wrong run"
assert get_input(receiver, "/inputs/p/a.json", task="done")[0] == 403, "a task that is not running"
assert get_input(receiver, "/inputs/p/a.json", task="idle")[0] == 403, "a running task without that input"
assert get_input(receiver, "/inputs/p/b.json")[0] == 403, "not among the task's inputs"
assert get_input(receiver, "/inputs/p/missing.json")[0] == 404, "declared but not delivered"
for escape in ("/inputs/../secret.txt", "/inputs/%2e%2e/secret.txt", "/inputs/p/../../secret.txt", "/inputs//etc/passwd"):
code, body = get_input(receiver, escape)
assert code in (403, 404) and b"outside" not in body, escape
assert get_input(receiver, "/p/a.json")[0] == 404, "only /inputs/ is served"
finally:
receiver.shutdown()
receiver.server_close()
def test_a_missing_producer_artifact_fails_the_consumer_instead_of_running_it(fake: Fake, tmp_path: Path) -> None: def test_a_missing_producer_artifact_fails_the_consumer_instead_of_running_it(fake: Fake, tmp_path: Path) -> None:
script = json.loads(json.dumps(SCRIPT)) script = json.loads(json.dumps(SCRIPT))
del script["repo-facts"]["files"]["facts.json"] # repo-facts reports done but never delivered facts.json del script["repo-facts"]["files"]["facts.json"] # repo-facts reports done but never delivered facts.json
@@ -996,7 +921,7 @@ def test_a_megabyte_producer_artifact_keeps_every_rendered_value_under_substrate
(tmp_path / "repo-facts" / "facts.json").write_text(json.dumps({"blob": "f" * (1 << 20)}), encoding="utf-8") (tmp_path / "repo-facts" / "facts.json").write_text(json.dumps({"blob": "f" * (1 << 20)}), encoding="utf-8")
run, manifests = reconciler.load_run(write_run(tmp_path, "big")) run, manifests = reconciler.load_run(write_run(tmp_path, "big"))
for entry in run.tasks: for entry in run.tasks:
docs = reconciler.render_task(run, entry, manifests, "http://h:1/", "10.0.0.1:80") docs = reconciler.render_task(run, entry, manifests)
task, workspaces = docs[-1], [d for d in docs if d["kind"] == "Workspace"] task, workspaces = docs[-1], [d for d in docs if d["kind"] == "Workspace"]
assert all(len(e["value"]) < SUBSTRATE_ENV_LIMIT for e in task["spec"]["env"]), entry.name assert all(len(e["value"]) < SUBSTRATE_ENV_LIMIT for e in task["spec"]["env"]), entry.name
assert len(yaml.safe_dump(task)) < SUBSTRATE_ENV_LIMIT, f"{entry.name}: AX_TASK_YAML" assert len(yaml.safe_dump(task)) < SUBSTRATE_ENV_LIMIT, f"{entry.name}: AX_TASK_YAML"
@@ -1080,15 +1005,3 @@ def test_a_tar_the_store_path_cannot_accept_fails_the_task(fake: Fake, tmp_path:
assert not (reconciler.run_dir("esc").parent / "escape.json").exists() assert not (reconciler.run_dir("esc").parent / "escape.json").exists()
def test_receiver_mode_still_runs_through_the_host_receiver(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""OUSAST_DELIVERY=receiver (group 2 only): the old path, with its http rule and OUSAST_ARTIFACT_URL, until 2.6."""
monkeypatch.setenv("OUSAST_DELIVERY", "receiver")
assert reconciler.run(write_run(tmp_path, "rcv"), ax=str(fake.ax)) == "done"
assert fake.applied("rcv", "verify")["env"]["OUSAST_ARTIFACT_URL"].startswith("http://127.0.0.1:")
assert [s["body"].get("delivery") for s in fake.router.starts] == [None, None, None], "no presigned URLs in receiver mode"
assert not any(k.startswith("runs/rcv/") and k.endswith("output.tar") for k in fake.store.objects), "nothing PUT to the store"
assert "runs/rcv/state.json" in fake.store.objects, "the state is still mirrored"
written = json.loads((tmp_path / "policies" / "default_rcv-repo-facts.written.json").read_text())
assert written["rules"] == [{"tlsPassthrough": {"hostnames": ["example.invalid"], "ports": {"numbers": [443]}}}], (
"an IP receiver: no http rule"
)
+1 -1
View File
@@ -39,7 +39,7 @@ def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]:
return rows return rows
def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None: def test_remember_turns_the_saved_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes() facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
assert len(facts) > 100, "the fixture was read" assert len(facts) > 100, "the fixture was read"
store = FileStore(tmp_path / "memory") store = FileStore(tmp_path / "memory")
+29 -217
View File
@@ -19,13 +19,12 @@ import time
import urllib.error import urllib.error
import urllib.request import urllib.request
from collections.abc import Iterator from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
import pytest import pytest
from plane_fake_store import HttpObjectStore from plane_fake_store import HttpObjectStore
from openultrasast.plane import egress, runner from openultrasast.plane import runner
from openultrasast.plane.manifests import GitSource from openultrasast.plane.manifests import GitSource
from openultrasast.plane.router import Router, StartError from openultrasast.plane.router import Router, StartError
from openultrasast.plane.runner import ( from openultrasast.plane.runner import (
@@ -194,96 +193,10 @@ def test_main_serves_readiness_on_a_free_port_and_the_task_sees_200(
# --- artifact delivery (Requirement 4.4) -------------------------------------------------------------------------------- # --- artifact delivery (Requirement 4.4) --------------------------------------------------------------------------------
class Receiver(ThreadingHTTPServer):
"""Records every POST (headers and body) and answers with ``status``."""
def __init__(self, status: int) -> None:
super().__init__(("127.0.0.1", 0), _ReceiverHandler)
self.status = status
self.posts: list[tuple[dict[str, str], bytes]] = []
self.thread = threading.Thread(target=self.serve_forever, daemon=True)
self.thread.start()
@property
def url(self) -> str:
return f"http://127.0.0.1:{self.server_address[1]}/artifacts"
def stop(self) -> None:
self.shutdown()
self.server_close()
class _ReceiverHandler(BaseHTTPRequestHandler):
server: Receiver
def do_POST(self) -> None: # noqa: N802
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
self.server.posts.append(({k: v for k, v in self.headers.items()}, body))
self.send_response(self.server.status)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
@pytest.fixture
def receiver() -> Iterator[Receiver]:
server = Receiver(200)
try:
yield server
finally:
server.stop()
def test_output_dir_is_delivered_as_a_tar_with_the_headers(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, OUSAST_RUN="validation-46")
assert code == 0
assert len(receiver.posts) == 1
headers, body = receiver.posts[0]
assert headers["Content-Type"] == "application/x-tar"
assert headers["X-Ousast-Task"] == "stub"
assert headers["X-Ousast-Run"] == "validation-46"
receiver.posts.clear()
again = {"OUSAST_ARTIFACT_URL": receiver.url, "OUSAST_TASK": "facts", "OUSAST_STATE_DIR": str(tmp_path / "again-state")}
code, _ = run_main(monkeypatch, tmp_path / "again", files_workspace_yaml(), **again)
assert code == 0 and receiver.posts[0][0]["X-Ousast-Task"] == "facts", "the Run's task name, not the ax Task's"
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as archive:
names = sorted(archive.getnames())
summary = archive.extractfile("summary.json")
assert summary is not None
assert json.loads(summary.read())["status"] == "done"
assert names == ["facts.json", "summary.json"]
def test_run_header_is_absent_without_ousast_run(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url)
assert code == 0
assert "X-Ousast-Run" not in receiver.posts[0][0]
def test_delivery_failure_is_reported_as_failed_with_exit_2(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
broken = Receiver(500)
try:
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url)
finally:
broken.stop()
assert code == EXIT_BY_STATUS["failed"] == 2
assert len(broken.posts) == 3, "three attempts"
summary = json.loads((output / "summary.json").read_text())
assert summary["status"] == "failed"
assert summary["reason"].startswith("artifact delivery failed: HTTP 500")
assert summary["units_done"] == 1 and summary["units_total"] == 1, "the task's other fields are kept"
def test_deliver_retries_on_connection_errors_then_raises() -> None: def test_deliver_retries_on_connection_errors_then_raises() -> None:
closed = Receiver(200) closed = HttpObjectStore()
url = closed.url url = closed.presign_put("output.tar")
closed.stop() closed.close()
with pytest.raises(DeliveryError, match="3 attempts"): with pytest.raises(DeliveryError, match="3 attempts"):
deliver(url, b"x", {"Content-Type": "application/x-tar"}, backoff=0) deliver(url, b"x", {"Content-Type": "application/x-tar"}, backoff=0)
@@ -366,9 +279,9 @@ def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name:
return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log) return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], tmp_path: Path) -> None:
port = free_port() port = free_port()
proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url) proc = spawn_runner(tmp_path, port)
marker = tmp_path / "state" / "stub.done.json" marker = tmp_path / "state" / "stub.done.json"
try: try:
wait_for(marker.exists) wait_for(marker.exists)
@@ -377,8 +290,7 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env:
base = f"http://127.0.0.1:{port}" base = f"http://127.0.0.1:{port}"
assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200 assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200
assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200 assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200
assert len(receiver.posts) == 1 assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": False, "summary": None}
assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": True, "summary": None}
proc.send_signal(signal.SIGTERM) proc.send_signal(signal.SIGTERM)
assert proc.wait(timeout=10) == 0 assert proc.wait(timeout=10) == 0
finally: finally:
@@ -386,9 +298,9 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env:
proc.wait() proc.wait()
def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], tmp_path: Path) -> None:
port = free_port() port = free_port()
proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="sleep", OUSAST_TERM_GRACE="1") proc = spawn_runner(tmp_path, port, STUB_MODE="sleep", OUSAST_TERM_GRACE="1")
pids_file = tmp_path / "out" / "pids.json" pids_file = tmp_path / "out" / "pids.json"
try: try:
wait_for(pids_file.exists) wait_for(pids_file.exists)
@@ -403,45 +315,21 @@ def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: di
proc.kill() proc.kill()
proc.wait() proc.wait()
assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete" assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete"
assert receiver.posts == [], "nothing is delivered for an interrupted command"
def test_the_completion_marker_prevents_a_rerun_on_resume( def test_the_completion_marker_prevents_a_rerun_on_resume(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None: ) -> None:
counter = tmp_path / "runs.txt" counter = tmp_path / "runs.txt"
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter))
assert code == 0 and len(counter.read_text().splitlines()) == 1 and len(receiver.posts) == 1 assert code == 0 and len(counter.read_text().splitlines()) == 1
(tmp_path / "ws" / "candidates.json").write_text("changed by the task\n") (tmp_path / "ws" / "candidates.json").write_text("changed by the task\n")
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter))
assert code == 0 assert code == 0
assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work" assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work"
assert len(receiver.posts) == 1, "a delivered result is not delivered again"
assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once" assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once"
def test_a_failed_delivery_is_retried_alone_on_the_next_boot(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
counter = tmp_path / "runs.txt"
broken = Receiver(500)
try:
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url, STUB_COUNTER=str(counter))
finally:
broken.stop()
assert code == 2
marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
assert marker["delivered"] is False and marker["status"] == "failed" and marker["exit"] == 0
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
assert code == 0
assert len(counter.read_text().splitlines()) == 1, "only the delivery is retried"
with tarfile.open(fileobj=io.BytesIO(receiver.posts[0][1]), mode="r:") as archive:
summary = archive.extractfile("summary.json")
assert summary is not None and json.loads(summary.read())["status"] == "done", "the task's own summary"
marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
assert marker["delivered"] is True and marker["status"] == "done"
# --- git workspaces from the case cache and by shallow clone ---------------------------------------------------------------- # --- git workspaces from the case cache and by shallow clone ----------------------------------------------------------------
@@ -590,14 +478,13 @@ def test_git_commits_annotation_round_trips_to_the_checkout(
"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata: {name: rt}\nspec: {tasks: [{name: stub, task: stub}]}\n" "apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata: {name: rt}\nspec: {tasks: [{name: stub, task: stub}]}\n"
) )
run, manifests = reconciler.load_run(manifest) run, manifests = reconciler.load_run(manifest)
*rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests, "unused") *rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests)
assert rest[0]["metadata"] == {"name": "case"}, "annotations are stripped before ax" assert rest[0]["metadata"] == {"name": "case"}, "annotations are stripped before ax"
assert rest[0]["spec"]["git"] == [{"name": "code", "repo": repo.as_uri(), "branch": "main"}], "no commit for ax" assert rest[0]["spec"]["git"] == [{"name": "code", "repo": repo.as_uri(), "branch": "main"}], "no commit for ax"
overrides = {"OUSAST_OUTPUT_DIR": str(output)} overrides = {"OUSAST_OUTPUT_DIR": str(output)}
task_doc["spec"]["env"] = [ task_doc["spec"]["env"] = [
{"name": e["name"], "value": overrides.get(e["name"], e["value"])} {"name": e["name"], "value": overrides.get(e["name"], e["value"])}
for e in task_doc["spec"]["env"] for e in task_doc["spec"]["env"]
if e["name"] != "OUSAST_ARTIFACT_URL"
] ]
assert json.loads({e["name"]: e["value"] for e in task_doc["spec"]["env"]}["OUSAST_GIT_PINS"]) == {"case/code": pin} assert json.loads({e["name"]: e["value"] for e in task_doc["spec"]["env"]}["OUSAST_GIT_PINS"]) == {"case/code": pin}
monkeypatch.setenv("AX_TASK_YAML", yaml.safe_dump(task_doc)) monkeypatch.setenv("AX_TASK_YAML", yaml.safe_dump(task_doc))
@@ -669,11 +556,11 @@ def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict
def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child( def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child(
env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture env: dict[str, str], tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
caplog.set_level(logging.DEBUG) caplog.set_level(logging.DEBUG)
counter = tmp_path / "runs.txt" counter = tmp_path / "runs.txt"
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) thread, environ, codes = main_in_thread(tmp_path, STUB_COUNTER=str(counter))
port = environ["AX_RUNNER_BOUND_PORT"] port = environ["AX_RUNNER_BOUND_PORT"]
time.sleep(0.5) time.sleep(0.5)
assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request" assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request"
@@ -693,15 +580,14 @@ def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only
assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there" assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there"
assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files" assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files"
assert files_containing(tmp_path, SECRET) == [], "no file holds the credential" assert files_containing(tmp_path, SECRET) == [], "no file holds the credential"
assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar"
assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record" assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record"
assert any("start request: 202" in record.getMessage() for record in caplog.records) assert any("start request: 202" in record.getMessage() for record in caplog.records)
def test_a_crash_that_prints_the_credential_is_redacted( def test_a_crash_that_prints_the_credential_is_redacted(
env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str] env: dict[str, str], tmp_path: Path, capfd: pytest.CaptureFixture[str]
) -> None: ) -> None:
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak") thread, environ, codes = main_in_thread(tmp_path, STUB_MODE="leak")
start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}} start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202 assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202
thread.join(timeout=30) thread.join(timeout=30)
@@ -727,23 +613,25 @@ def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch:
assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists() assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists()
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: def test_the_golden_boot_never_runs_the_command(env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path) -> None:
"""Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed """Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed
actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start.""" actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start."""
counter = tmp_path / "runs.txt" counter = tmp_path / "runs.txt"
same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"} same = {"STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1", "AX_RUNNER_INSECURE_DELIVERY": "1"}
real_port, golden_port = free_port(), free_port() real_port, golden_port = free_port(), free_port()
real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same) real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same)
golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same) golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same)
try: try:
for port in (real_port, golden_port): for port in (real_port, golden_port):
wait_for(lambda port=port: ready_on(port)) wait_for(lambda port=port: ready_on(port))
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}) Router(f"http://127.0.0.1:{real_port}").start_task(
"default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}, {"put": object_store.presign_put("output.tar")}
)
marker = tmp_path / "state" / "stub.done.json" marker = tmp_path / "state" / "stub.done.json"
wait_for(marker.exists) wait_for(marker.exists)
time.sleep(1.0) time.sleep(1.0)
assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command" assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command"
assert golden.poll() is None and len(receiver.posts) == 1 assert golden.poll() is None
with pytest.raises(StartError, match="already started"): with pytest.raises(StartError, match="already started"):
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {}) Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {})
again_port = free_port() again_port = free_port()
@@ -762,10 +650,10 @@ def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: R
assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential" assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"
def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], tmp_path: Path) -> None:
"""Substrate's golden boot never gets a start, so it must touch no workspace (and no network): a clone at boot """Substrate's golden boot never gets a start, so it must touch no workspace (and no network): a clone at boot
failed on the live cluster while the actor was still being restored (2026-09-29).""" failed on the live cluster while the actor was still being restored (2026-09-29)."""
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url) thread, environ, codes = main_in_thread(tmp_path)
time.sleep(0.5) time.sleep(0.5)
written = [p for p in (tmp_path / "ws").rglob("*") if p.is_file()] if (tmp_path / "ws").exists() else [] written = [p for p in (tmp_path / "ws").rglob("*") if p.is_file()] if (tmp_path / "ws").exists() else []
assert written == [], "nothing is materialised before the start request" assert written == [], "nothing is materialised before the start request"
@@ -778,82 +666,6 @@ def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str
# --- inputs other tasks produced: fetched from the receiver after the start (Requirement 3.1, 4.4) ------------------- # --- inputs other tasks produced: fetched from the receiver after the start (Requirement 3.1, 4.4) -------------------
@pytest.fixture
def plane_receiver(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[egress.Receiver]:
"""The reconciler's own receiver on a free port, with a producer's artifact delivered and the task ``stub`` running."""
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
server = egress.Receiver("r", tmp_path / "run", 0)
(tmp_path / "run" / "repo-facts").mkdir(parents=True)
(tmp_path / "run" / "repo-facts" / "facts.json").write_bytes(b'{"a.py": ["f"]}' * 5000)
server.running.add("stub")
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield server
finally:
server.shutdown()
server.server_close()
def input_env(server: egress.Receiver, tmp_path: Path) -> dict[str, str]:
return {
"OUSAST_ARTIFACT_URL": server.url,
"OUSAST_RUN": "r",
"OUSAST_TASK": "stub",
"OUSAST_INPUTS": json.dumps({"FACTS": "repo-facts/facts.json"}),
"OUSAST_INPUT_FACTS": str(tmp_path / "ws" / "in" / "repo-facts" / "facts.json"),
}
def test_inputs_are_fetched_from_the_receiver_before_the_command(
env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
plane_receiver.inputs["stub"] = frozenset({"repo-facts/facts.json"})
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path))
assert code == 0
fetched = tmp_path / "ws" / "in" / "repo-facts" / "facts.json"
assert fetched.read_bytes() == (tmp_path / "run" / "repo-facts" / "facts.json").read_bytes()
assert "in/repo-facts/facts.json" in json.loads((output / "facts.json").read_text())["workspace_files"], "there before the command ran"
assert "stub" in plane_receiver.delivered
def test_an_input_that_cannot_be_fetched_delivers_a_failed_summary_and_never_runs_the_command(
env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
plane_receiver.inputs["stub"] = frozenset({"repo-facts/other.json"}) # facts.json is not among its inputs: 403
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path))
assert code == EXIT_BY_STATUS["failed"]
assert not (output / "facts.json").exists(), "the command never ran with an input missing"
delivered = json.loads((tmp_path / "run" / "stub" / "summary.json").read_text())
assert delivered["status"] == "failed" and "input FACTS (repo-facts/facts.json) not fetched: HTTP 403" in delivered["reason"]
assert not (tmp_path / "ws" / "in" / "repo-facts").exists() or not any((tmp_path / "ws" / "in" / "repo-facts").iterdir())
def test_fetch_dials_the_override_and_sends_the_url_host(tmp_path: Path) -> None:
seen: list[dict[str, str]] = []
class _Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
seen.append({"path": self.path, **dict(self.headers.items())})
self.send_response(200)
self.send_header("Content-Length", "2")
self.end_headers()
self.wfile.write(b"ok")
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
dial = f"127.0.0.1:{server.server_address[1]}"
size = runner.fetch("http://receiver.example/inputs/p/a.json", {"X-Ousast-Run": "r"}, tmp_path / "x" / "a.json", dial=dial)
finally:
server.shutdown()
server.server_close()
assert size == 2 and (tmp_path / "x" / "a.json").read_bytes() == b"ok"
assert seen[0]["path"] == "/inputs/p/a.json" and seen[0]["Host"] == "receiver.example" and seen[0]["X-Ousast-Run"] == "r"
# --- store delivery through the start body (plane-on-kubernetes 2.2, design section 3) --------------------------- # --- store delivery through the start body (plane-on-kubernetes 2.2, design section 3) ---------------------------
@@ -905,7 +717,7 @@ def test_output_is_put_to_the_presigned_url_and_inputs_are_fetched_from_theirs(
env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path, caplog: pytest.LogCaptureFixture env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
"""End to end against the S3-shaped fake: the start carries the URLs, the runner PUTs the tar and GETs the input; """End to end against the S3-shaped fake: the start carries the URLs, the runner PUTs the tar and GETs the input;
no receiver, no OUSAST_ARTIFACT_URL, no URL in any log line.""" no URL in any log line."""
caplog.set_level(logging.DEBUG) caplog.set_level(logging.DEBUG)
object_store._put("runs/run-1/repo-facts/facts.json", b'{"a.py": ["f"]}') object_store._put("runs/run-1/repo-facts/facts.json", b'{"a.py": ["f"]}')
put_url = object_store.presign_put("runs/run-1/stub/output.tar") put_url = object_store.presign_put("runs/run-1/stub/output.tar")
@@ -964,9 +776,9 @@ def test_a_failed_put_is_retried_alone_on_the_next_start_with_a_fresh_url(
def test_deliver_puts_and_names_only_the_safe_url_on_failure(object_store: HttpObjectStore) -> None: def test_deliver_puts_and_names_only_the_safe_url_on_failure(object_store: HttpObjectStore) -> None:
url = object_store.presign_put("k/output.tar") url = object_store.presign_put("k/output.tar")
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, method="PUT") == 200 assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0) == 200
assert object_store.objects["k/output.tar"] == b"tar" assert object_store.objects["k/output.tar"] == b"tar"
with pytest.raises(runner.DeliveryError) as caught: with pytest.raises(runner.DeliveryError) as caught:
runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0, method="PUT") runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0)
assert "HTTP 404" in str(caught.value) and "X-Amz" not in str(caught.value) and "/other/k/output.tar" in str(caught.value) assert "HTTP 404" in str(caught.value) and "X-Amz" not in str(caught.value) and "/other/k/output.tar" in str(caught.value)
assert runner.safe_url("https://h:1/p/q?X-Amz-Signature=s") == "https://h:1/p/q" assert runner.safe_url("https://h:1/p/q?X-Amz-Signature=s") == "https://h:1/p/q"
+3 -3
View File
@@ -6,7 +6,7 @@ import subprocess
import pytest import pytest
from openultrasast.push_inputs import validate_manifest from benchmarks.push.push_inputs import validate_manifest
def git(root, *args): def git(root, *args):
@@ -118,7 +118,7 @@ def test_authored_edit_requires_exact_original_and_provenance(manifest):
@pytest.mark.parametrize("field", ["snapshots", "cases"]) @pytest.mark.parametrize("field", ["snapshots", "cases"])
def test_empty_population_rejected(manifest, field): def test_empty_population_rejected(manifest, field):
from openultrasast.push_inputs import InputError from benchmarks.push.push_inputs import InputError
data, _, run = manifest data, _, run = manifest
data[field] = [] data[field] = []
@@ -127,7 +127,7 @@ def test_empty_population_rejected(manifest, field):
def test_missing_snapshot_identity_rejected(manifest): def test_missing_snapshot_identity_rejected(manifest):
from openultrasast.push_inputs import InputError from benchmarks.push.push_inputs import InputError
data, _, run = manifest data, _, run = manifest
del data["snapshots"][0]["id"] del data["snapshots"][0]["id"]
+1 -21
View File
@@ -5,14 +5,7 @@ import pytest
from openultrasast.preprocess import FileTarget from openultrasast.preprocess import FileTarget
from openultrasast.provider.openrouter import OpenRouterError, parse_json_content from openultrasast.provider.openrouter import OpenRouterError, parse_json_content
from openultrasast.rank import ( from openultrasast.rank import RankingError, composite_priority, parse_ranker_response, rank_targets, write_rankings
RankingError,
composite_priority,
parse_ranker_response,
rank_targets,
rank_targets_with_model,
write_rankings,
)
class FakeRankerClient: class FakeRankerClient:
@@ -72,19 +65,6 @@ def test_parse_ranker_response_rejects_malformed_scores() -> None:
parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model") parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model")
def test_rank_targets_with_model_chunks_requests() -> None:
targets = [
FileTarget(path="a.py", absolute_path="/repo/a.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
FileTarget(path="b.py", absolute_path="/repo/b.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
]
client = FakeRankerClient()
rankings = rank_targets_with_model(targets, client=client, model="openrouter/test-ranker", chunk_size=1)
assert client.calls == 2
assert {ranking.model_id for ranking in rankings} == {"openrouter/test-ranker"}
def test_parse_json_content_accepts_fenced_json() -> None: def test_parse_json_content_accepts_fenced_json() -> None:
assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []} assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []}
-27
View File
@@ -169,33 +169,6 @@ def test_claim_is_visible_in_worker_snapshot(tmp_path):
Coordinator(board(tmp_path), execute, budget=SearchBudget(reason_rounds=1)).run() Coordinator(board(tmp_path), execute, budget=SearchBudget(reason_rounds=1)).run()
def test_batch_adapter_offline_checkpoint_and_step_dispatch(tmp_path, monkeypatch):
from dataclasses import asdict
from pathlib import Path
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1]))
from openultrasast.search.coordinator import BatchExecutor, SearchTask
calls = []
def lane(item, output, attempt):
import json
spec = json.loads(item.inputs["SPEC_URL"])
calls.append(spec)
assert item.extra_env["SEARCH_STEP"] == spec["step"]
return {"status": "ok", "cost_usd": 0, "intents": []}
adapter = BatchExecutor(
image="registry/task@sha256:" + "a" * 64, command=("python3", "/worker.py"), output=tmp_path, lanes={"ax": lane}
)
for index, step in enumerate(("reason", "explore", "verify")):
task = SearchTask(f"task-{index}", "search-1", step, "facts: []", {}, asdict(SearchBudget()))
assert adapter(task)["status"] == "ok"
assert adapter(task)["status"] == "ok"
assert len(calls) == 3
def test_invalid_identifiers_refused_before_paths(tmp_path): def test_invalid_identifiers_refused_before_paths(tmp_path):
from dataclasses import asdict from dataclasses import asdict
-11
View File
@@ -10,22 +10,11 @@ from openultrasast.verification import (
VerificationStatus, VerificationStatus,
build_independent_verifier_context, build_independent_verifier_context,
is_report_verified, is_report_verified,
validate_evidence_transition,
verify_finding, verify_finding,
write_verification_results, write_verification_results,
) )
def test_evidence_transition_rejects_backward_and_skipped_levels() -> None:
assert validate_evidence_transition("suspicion", "static_corroboration") == EvidenceLevel.STATIC_CORROBORATION
with pytest.raises(ValueError, match="backward"):
validate_evidence_transition("root_cause_explained", "static_corroboration")
with pytest.raises(ValueError, match="skip"):
validate_evidence_transition("suspicion", "root_cause_explained")
def test_verified_status_requires_static_corroboration_and_acceptance() -> None: def test_verified_status_requires_static_corroboration_and_acceptance() -> None:
assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED) assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED)
assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED) assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED)