From f63703a6042d911fc551b30f55baee9d89a552ca Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:06:09 +0200 Subject: [PATCH] chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 --- .kiro/breadth.toml | 15 +- benchmarks/push/README.md | 2 +- benchmarks/push/m1a_record_vetoes.py | 2 +- benchmarks/push/measure.py | 4 +- .../push}/push_inputs.py | 0 benchmarks/push/transfer_replay.py | 2 +- docs/memory-and-detection.md | 6 +- ops/ax/smoke-run.sh | 4 - src/openultrasast/calibration.py | 42 --- src/openultrasast/cli.py | 2 +- src/openultrasast/findings.py | 11 - src/openultrasast/fusion.py | 7 +- src/openultrasast/harness.py | 78 +----- src/openultrasast/hunter_tools.py | 240 +---------------- src/openultrasast/learn/features.py | 140 +--------- src/openultrasast/learn/folds.py | 36 +-- src/openultrasast/learn/roles.py | 7 +- src/openultrasast/mapping.py | 77 ------ src/openultrasast/model/config_value.py | 8 +- src/openultrasast/model/pipeline.py | 27 +- src/openultrasast/model/report.py | 82 +----- src/openultrasast/model/taint.py | 49 +--- src/openultrasast/model/taxonomy.py | 21 -- src/openultrasast/plane/delivery.py | 6 - src/openultrasast/plane/egress.py | 173 +----------- src/openultrasast/plane/reconciler.py | 46 +--- src/openultrasast/plane/runner.py | 74 ++---- src/openultrasast/plane/tasks/features.py | 18 +- src/openultrasast/rank.py | 52 +--- src/openultrasast/search/coordinator.py | 82 ------ src/openultrasast/search/verify.py | 19 +- src/openultrasast/search/worker.py | 46 +--- src/openultrasast/tool_hunter.py | 82 +----- src/openultrasast/verification.py | 10 - tests/test_calibration.py | 29 --- tests/test_findings.py | 16 +- tests/test_fusion.py | 16 +- tests/test_harness.py | 72 +---- tests/test_hunter_curated_tools.py | 198 -------------- tests/test_hunter_tools.py | 42 +-- tests/test_learn_folds.py | 27 +- tests/test_learn_retrieve.py | 21 +- tests/test_learn_roles.py | 18 +- tests/test_learn_schema.py | 58 +---- tests/test_mapping.py | 56 ---- tests/test_model_config_value.py | 48 ++-- tests/test_model_report.py | 90 ------- tests/test_model_taint.py | 88 +++---- tests/test_model_taxonomy.py | 16 -- tests/test_plane_delivery.py | 10 - tests/test_plane_egress.py | 61 +---- tests/test_plane_features.py | 35 +-- tests/test_plane_increment.py | 2 +- tests/test_plane_reconciler.py | 101 +------ tests/test_plane_remember.py | 2 +- tests/test_plane_runner.py | 246 +++--------------- tests/test_push_inputs.py | 6 +- tests/test_rank.py | 22 +- tests/test_search_coordinator.py | 27 -- tests/test_verification.py | 11 - 60 files changed, 212 insertions(+), 2576 deletions(-) rename {src/openultrasast => benchmarks/push}/push_inputs.py (100%) delete mode 100644 tests/test_hunter_curated_tools.py delete mode 100644 tests/test_model_report.py diff --git a/.kiro/breadth.toml b/.kiro/breadth.toml index 34b5455..4f8bcdf 100644 --- a/.kiro/breadth.toml +++ b/.kiro/breadth.toml @@ -8,6 +8,7 @@ # parked = no work lands; scaffolding (stubs, unused flags, dead task modules) is removed; spec stays as history # retiring = the "closes" deletion is scheduled; the spec directory goes with it # review = Pathfinder 2026-10-13 decides between parked and retiring +# removed = the "closes" deletion has run; the entry stays one release as the record of what went # # open_fronts (the weekly number) = specs with status active or review # + capabilities with status review or retiring whose "closes" has not run. @@ -181,7 +182,7 @@ status = "active" spec = "hunter-learning-loop" modules = ["tool_hunter.py", "hunter_tools.py", "provider/openrouter.py", "model/endpoint.py", "plane/budget.py"] lines = 1700 -note = "Pathfinder U6 counts the chat-client classes; expected outcome: one client protocol, one metering path" +note = "four never-enabled hunter tools removed 2026-10-13 (P0); P2 collapses the client stack to one client, one meter, one resolver" [capability.learn] status = "active" @@ -198,7 +199,7 @@ lines = 4803 [capability.core] status = "active" modules = ["cli.py", "config.py", "contracts.py", "preprocess.py", "findings.py", "rank.py", "mapping.py", "ruleset", "policy", "reports.py", "stages.py", "run.py", "provenance.py", "profiling.py"] -note = "hunter.py is the regex quick-rule pool and should be renamed; harness.py processor machinery is unreachable (U1)" +note = "hunter.py is the regex quick-rule pool and should be renamed" [capability.pairs-and-benchmarks] status = "active" @@ -214,13 +215,13 @@ lines = 4885 closes = "Pathfinder U8 lists which modules only the retired HarnessX loop used (engine_alerts, harvest, workspaces, profile are the suspects); delete those" [capability.regress] -status = "review" +status = "removed" # f5a904b, 2026-10-13: deep mode records regress_retired; proof is the search line modules = ["regress"] lines = 549 closes = "Pathfinder U9: if search/ covers the triggerable verdict, delete regress/ and the deep-mode REGRESS branch in cli.py:839-880" [capability.improve] -status = "retiring" +status = "removed" # 94e96c6, 2026-10-13 modules = ["improve"] lines = 1175 note = "HarnessX-era improve lever; no active spec names it" @@ -228,13 +229,13 @@ closes = "delete improve/, `ousast improve` (cli.py:237, 433, 1106), plane/tasks [capability.scan-extras] status = "review" -modules = ["fusion.py", "verification.py", "calibration.py", "scoring", "harness.py (processors)"] +modules = ["fusion.py", "verification.py", "calibration.py", "scoring"] # harness.py processor machinery removed 2026-10-13 closes = "U1 found fusion decisions never reach findings or score, verification runs before MAP, processors unreachable; decide keep-and-fix or delete per module" [capability.mcp-and-skills] status = "parked" spec = "mcp-shift-left-agent" -modules = ["mcp.py", "skills.py", "index.py"] +modules = ["mcp.py"] # skills.py and index.py removed f5a904b, 2026-10-13 lines = 830 closes = "see spec.mcp-shift-left-agent" @@ -246,6 +247,6 @@ closes = "U10: keep gate.py (CI + plane loop); delete map_gate.py and pair_gate. [capability.push-evaluation-instruments] status = "review" -modules = ["push_inputs.py", "push_scoring.py"] +modules = ["push_scoring.py"] # push_inputs.py moved to benchmarks/push 2026-10-13; push_scoring stays (eligibility.py:18) lines = 462 closes = "U3: move under benchmarks/push or delete if only tests use them" diff --git a/benchmarks/push/README.md b/benchmarks/push/README.md index 37abd52..e05fbf4 100644 --- a/benchmarks/push/README.md +++ b/benchmarks/push/README.md @@ -1,7 +1,7 @@ # Reproducible push evaluation inputs `inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run. -Run `.venv/bin/python -m openultrasast.push_inputs benchmarks/push/inputs.json`. +Run `.venv/bin/python -m benchmarks.push.push_inputs benchmarks/push/inputs.json`. It is offline by default, even if the ordinary repository network environment variable is set. `--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits. No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one diff --git a/benchmarks/push/m1a_record_vetoes.py b/benchmarks/push/m1a_record_vetoes.py index 4b9423a..37ed194 100644 --- a/benchmarks/push/m1a_record_vetoes.py +++ b/benchmarks/push/m1a_record_vetoes.py @@ -26,11 +26,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from measure import git, pin_snapshot # noqa: E402 +from benchmarks.push.push_inputs import validate_manifest # noqa: E402 from openultrasast.model.scan import ScanBudget # noqa: E402 from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402 from openultrasast.push.runner import _provenance # noqa: E402 from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402 -from openultrasast.push_inputs import validate_manifest # noqa: E402 from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402 TARGET = {"path": "app/routes/contributions.js", "function": "handleContributionsUpdate", "family": "injection"} diff --git a/benchmarks/push/measure.py b/benchmarks/push/measure.py index eb80f71..3aa3366 100644 --- a/benchmarks/push/measure.py +++ b/benchmarks/push/measure.py @@ -13,10 +13,10 @@ import time from dataclasses import replace from pathlib import Path +from benchmarks.push.push_inputs import validate_manifest from openultrasast.model.contracts import QuestionIdentity from openultrasast.model.scan import ScanBudget from openultrasast.push.runner import _provenance -from openultrasast.push_inputs import validate_manifest from openultrasast.push_scoring import freeze_profile, score from openultrasast.repos import checkout_path, load_repo_recipe @@ -314,7 +314,7 @@ def main() -> int: "reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.", "selection": args.selection, "analysis_deadline_seconds": args.deadline, - "recorded_cases": len(run["records"]), + "case_count": len(run["records"]), "declared_population": len(manifest["cases"]), "core_unchanged": True, "adapter_changes": [], diff --git a/src/openultrasast/push_inputs.py b/benchmarks/push/push_inputs.py similarity index 100% rename from src/openultrasast/push_inputs.py rename to benchmarks/push/push_inputs.py diff --git a/benchmarks/push/transfer_replay.py b/benchmarks/push/transfer_replay.py index b35e7c8..9ea801b 100644 --- a/benchmarks/push/transfer_replay.py +++ b/benchmarks/push/transfer_replay.py @@ -31,11 +31,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from measure import git, pin_snapshot # noqa: E402 +from benchmarks.push.push_inputs import validate_manifest # noqa: E402 from openultrasast.model.scan import ScanBudget # noqa: E402 from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402 from openultrasast.push.runner import _provenance # noqa: E402 from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402 -from openultrasast.push_inputs import validate_manifest # noqa: E402 from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402 # What each declared comparison is expected to show, stated before anything runs. `introduce` diff --git a/docs/memory-and-detection.md b/docs/memory-and-detection.md index d6c3dc0..b15f1bb 100644 --- a/docs/memory-and-detection.md +++ b/docs/memory-and-detection.md @@ -26,11 +26,7 @@ flowchart TD op -- "otherwise" --> adv["ADVISORY"] ``` -These decisions are meant for changed code. `learn/features.py` builds feature records in two ways: - -- for a whole-repository scan; -- for a base..head delta (`build_for_scan` with `Delta`). This is the same delta `ousast pre-push` checks. - +The plane task uses `learn/features.py` to build candidate feature records from delivered artifacts. Integration into scan and pre-push is still ahead. ## 1. Instruments produce signals, not verdicts diff --git a/ops/ax/smoke-run.sh b/ops/ax/smoke-run.sh index 09c7c80..c4ada17 100755 --- a/ops/ax/smoke-run.sh +++ b/ops/ax/smoke-run.sh @@ -8,10 +8,6 @@ REPO="$(cd "$(dirname "$0")/../.." && pwd)" PY="${OUSAST_PYTHON:-$REPO/.venv/bin/python}" RUNNER_IMAGE="$(PYTHONPATH="$REPO/src" "$PY" -m openultrasast.plane.profile --profile "${OUSAST_PLANE_PROFILE:-kind}" --print images.runner)" export RUNNER_IMAGE -# Actors reach the receiver by its Service name through the egress gateway (ops/ax/receiver-service.yaml.tmpl, -# applied by up.sh, endpoint this host at OUSAST_ARTIFACT_PORT): the runner dials the Service's ClusterIP on port -# 80 with that name as Host; the reconciler looks the ClusterIP up unless OUSAST_ARTIFACT_DIAL is set. -export OUSAST_ARTIFACT_PORT="${OUSAST_ARTIFACT_PORT:-18090}" envsubst < "$(dirname "$0")/smoke-run.yaml.tmpl" > "$SRC/smoke-run.yaml" ousast() { PYTHONPATH="$REPO/src" "$PY" -m openultrasast.cli "$@"; } # this checkout's code, whatever python runs it ousast plane run "$SRC/smoke-run.yaml" "$@" diff --git a/src/openultrasast/calibration.py b/src/openultrasast/calibration.py index 1735db7..2c599ff 100644 --- a/src/openultrasast/calibration.py +++ b/src/openultrasast/calibration.py @@ -47,16 +47,6 @@ class FalsePositiveLearning: demotion: float = 0.5 -@dataclass(frozen=True) -class RankingMetrics: - tier: str - verified_findings: int - false_positives: int - time_to_verification_seconds: float - missed_fixture_vulnerabilities: int - false_positive_reasons: dict[str, int] - - @dataclass(frozen=True) class RankingCalibration: path: str @@ -105,38 +95,6 @@ def learning_from_finding_outcome( ) -def compute_ranking_metrics(events: list[dict[str, object]], *, tier: str) -> RankingMetrics: - verified = 0 - false_positives = 0 - reason_counts: dict[str, int] = {} - verification_times: list[float] = [] - missed = 0 - for event in events: - if event.get("tier") != tier: - continue - outcome = event.get("outcome") - if outcome == FindingOutcome.VERIFIED: - verified += 1 - if outcome in _FALSE_POSITIVE_OUTCOMES: - false_positives += 1 - reason = str(event.get("reason", FalsePositiveReason.UNVERIFIED)) - reason_counts[reason] = reason_counts.get(reason, 0) + 1 - duration = event.get("time_to_verification_seconds") - if isinstance(duration, int | float) and not isinstance(duration, bool): - verification_times.append(float(duration)) - missed_value = event.get("missed_fixture_vulnerabilities", 0) - if isinstance(missed_value, int) and not isinstance(missed_value, bool): - missed += missed_value - return RankingMetrics( - tier=tier, - verified_findings=verified, - false_positives=false_positives, - time_to_verification_seconds=round(sum(verification_times) / len(verification_times), 2) if verification_times else 0.0, - missed_fixture_vulnerabilities=missed, - false_positive_reasons=dict(sorted(reason_counts.items())), - ) - - def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearning]) -> RankingCalibration: applicable = [learning for learning in learnings if _learning_applies(ranking.path, learning)] calibrated = ranking.priority diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 6d490c5..4db4550 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -548,7 +548,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut config=config, trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets), ) - write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json") + write_harness_config(config=config, path=run.root / "harness.json") runtime.start(mode=mode, target=run.target) policy = runtime.run_stage("policy_load", load_policy) ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json" diff --git a/src/openultrasast/findings.py b/src/openultrasast/findings.py index 57dbaf8..d6fb6dd 100644 --- a/src/openultrasast/findings.py +++ b/src/openultrasast/findings.py @@ -99,17 +99,6 @@ def uncovered_languages( return sorted(counts.items()) -def build_quick_hunter_prompt(target: FileTarget, source_excerpt: str) -> str: - return ( - "Review this file for security-relevant issues. Return structured findings only when evidence exists.\n" - f"Path: {target.path}\n" - f"Language: {target.language}\n" - f"Tags: {', '.join(target.tags) or 'none'}\n" - "Source excerpt:\n" - f"{source_excerpt[:4000]}" - ) - - def write_findings(findings: list[StaticFinding], path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) payload = {"findings": [asdict(finding) for finding in findings]} diff --git a/src/openultrasast/fusion.py b/src/openultrasast/fusion.py index 0259e88..225c991 100644 --- a/src/openultrasast/fusion.py +++ b/src/openultrasast/fusion.py @@ -91,10 +91,6 @@ def fusion_triggers( return sorted(triggers) -def should_fuse(finding: StaticFinding, verification: VerificationResult | None = None, *, high_assurance: bool = False) -> bool: - return bool(fusion_triggers(finding, verification, high_assurance=high_assurance)) - - def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict: """Vulnerability-leaning panel: steel-mans the vulnerability case.""" reachable = finding.reachability_status == "reachable" @@ -213,5 +209,4 @@ __all__ = [ "fuse_finding", "fuse_findings", "fusion_triggers", - "should_fuse", -] + ] diff --git a/src/openultrasast/harness.py b/src/openultrasast/harness.py index 772b1d3..186c6b0 100644 --- a/src/openultrasast/harness.py +++ b/src/openultrasast/harness.py @@ -6,7 +6,7 @@ from collections.abc import Callable from dataclasses import asdict, dataclass, field from datetime import UTC, datetime from pathlib import Path -from typing import Any, Literal, Protocol +from typing import Any, Literal from .config import ResolvedConfig @@ -21,11 +21,6 @@ EventType = Literal[ "task_end", ] -ContractMode = Literal["strict", "warn"] - - -class HarnessContractError(ValueError): - """Raised when a processor violates its declared state contract.""" @dataclass(frozen=True) @@ -37,27 +32,9 @@ class HarnessEvent: payload: dict[str, Any] = field(default_factory=dict) -@dataclass(frozen=True) -class ProcessorSpec: - name: str - version: str - handles: tuple[EventType, ...] - reads: tuple[str, ...] = () - writes: tuple[str, ...] = () - - -class Processor(Protocol): - spec: ProcessorSpec - - def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any] | None: - raise NotImplementedError - - @dataclass(frozen=True) class SerializedHarnessConfig: - contract_mode: ContractMode model_roles: dict[str, str | None] - processor_versions: dict[str, str] prompt_hashes: dict[str, str] @@ -84,14 +61,10 @@ class HarnessRuntime: scan_id: str, config: ResolvedConfig, trace_writer: HarnessTraceWriter, - processors: list[Processor] | None = None, - contract_mode: ContractMode = "strict", ) -> None: self.scan_id = scan_id self.config = config self.trace_writer = trace_writer - self.processors = processors or [] - self.contract_mode = contract_mode self.state: dict[str, Any] = {"degradations": []} def start(self, *, mode: str, target: Path) -> None: @@ -119,71 +92,22 @@ class HarnessRuntime: payload=payload, ) self.trace_writer.append(event) - for processor in self.processors: - if event_type in processor.spec.handles: - self._run_processor(processor, event) - - def _run_processor(self, processor: Processor, event: HarnessEvent) -> None: - before = set(self.state) - missing_reads = [key for key in processor.spec.reads if key not in self.state] - if missing_reads: - self._contract_violation(processor.spec.name, f"missing reads: {', '.join(missing_reads)}") - return - - updates = processor.handle(event, dict(self.state)) or {} - if not isinstance(updates, dict): - self._contract_violation(processor.spec.name, "processor returned a non-dict update") - return - undeclared = sorted(set(updates) - set(processor.spec.writes)) - if undeclared: - self._contract_violation(processor.spec.name, f"undeclared writes: {', '.join(undeclared)}") - return - self.state.update(updates) - changed = sorted(set(self.state) - before | set(updates)) - self.trace_writer.append( - HarnessEvent( - event_type=event.event_type, - scan_id=self.scan_id, - stage=event.stage, - timestamp=datetime.now(UTC).isoformat(), - payload={"processor": processor.spec.name, "changed_state": changed}, - ) - ) - - def _contract_violation(self, processor_name: str, reason: str) -> None: - message = f"processor {processor_name} contract violation: {reason}" - if self.contract_mode == "strict": - raise HarnessContractError(message) - self.state.setdefault("degradations", []).append(message) - self.trace_writer.append( - HarnessEvent( - event_type="stage_end", - scan_id=self.scan_id, - stage="processor_contract", - timestamp=datetime.now(UTC).isoformat(), - payload={"status": "degraded", "reason": message}, - ) - ) def write_harness_config( *, config: ResolvedConfig, - processors: list[Processor], - contract_mode: ContractMode, path: Path, prompts: dict[str, str] | None = None, ) -> None: prompts = prompts or {} serialized = SerializedHarnessConfig( - contract_mode=contract_mode, model_roles={ "ranker": config.models.ranker, "hunter": config.models.hunter, "verifier": None, # schema kept byte for byte; LLM verification runs on the plane "patcher": None, # schema kept; `[models] patcher` was retired 2026-10-02 (never read) }, - processor_versions={processor.spec.name: processor.spec.version for processor in processors}, prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())}, ) path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/openultrasast/hunter_tools.py b/src/openultrasast/hunter_tools.py index ec348a6..195346f 100644 --- a/src/openultrasast/hunter_tools.py +++ b/src/openultrasast/hunter_tools.py @@ -2,10 +2,10 @@ from __future__ import annotations import os import re -from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Iterator from pathlib import Path -from .preprocess import FileTarget, enumerate_source_files +from .preprocess import enumerate_source_files class PathEscapesRepo(ValueError): @@ -80,25 +80,9 @@ def grep_repo(root: Path, pattern: str, *, max_matches: int) -> list[dict[str, o return matches -def find_refs(root: Path, symbol: str, mapping_index: object) -> list[dict[str, object]]: +def find_refs(root: Path, symbol: str) -> list[dict[str, object]]: resolved_root = root.resolve() refs: list[dict[str, object]] = [] - for record in _iter_mapping_records(mapping_index): - if not _record_matches_symbol(record, symbol): - continue - raw_path = _record_path(record) - if raw_path is None: - continue - clamped = clamp_repo_path(resolved_root, raw_path) - name = _record_value(record, "name", "function_name", "symbol") - refs.append( - { - "path": clamped.relative_to(resolved_root).as_posix(), - "name": str(name) if name is not None else symbol, - "line": _as_line(_record_value(record, "line")), - "source": "mapping", - } - ) if symbol: compiled = re.compile(rf"\b{re.escape(symbol)}\b") for relative, lines in _source_lines(resolved_root): @@ -113,221 +97,3 @@ def _iter_clamped_source_files(root: Path) -> Iterator[tuple[str, Path]]: for path in enumerate_source_files(resolved_root): relative = path.relative_to(resolved_root).as_posix() yield relative, clamp_repo_path(resolved_root, relative) - - -def _iter_mapping_records(mapping_index: object) -> Iterable[object]: - if mapping_index is None: - return () - if isinstance(mapping_index, Mapping): - return (mapping_index,) - if isinstance(mapping_index, (str, bytes, bytearray)): - return () - if isinstance(mapping_index, Iterable): - return mapping_index - return (mapping_index,) - - -def _record_matches_symbol(record: object, symbol: str) -> bool: - for field in ("name", "function_name", "symbol"): - value = _record_value(record, field) - if value is not None and str(value) == symbol: - return True - return False - - -def _record_path(record: object) -> str | None: - value = record.get("path") if isinstance(record, Mapping) else getattr(record, "path", None) - if value is None: - return None - return str(value) - - -def _record_value(record: object, *names: str) -> object | None: - for name in names: - raw: object | None - if isinstance(record, Mapping): - if name not in record: - continue - raw = record[name] - else: - raw = getattr(record, name, None) - if raw is not None: - return raw - return None - - -def _as_line(value: object) -> int | None: - if isinstance(value, bool) or value is None: - return None - if isinstance(value, int): - return value - return None - - -# --- curated structural tools (learning-harness, Req 6.4, 6.5) -------------------------------------- -# A detector reads code with `read_file` and `read_definition`; the three list tools below answer with -# identifiers and line spans only, so they can never become a second way to dump the repository. Every -# path a caller supplies is clamped to the scan root first. No caching: a detector calls these a handful -# of times per region, and a stale view of a tree that a round may have rewritten is worse than a reparse. - - -def read_definition(root: Path, symbol: str, *, max_chars: int, path: str | None = None) -> dict[str, object] | None: - """The definition of one named function: where it is, how far it runs, and its text. None when absent.""" - resolved_root = root.resolve() - scope = clamp_repo_path(resolved_root, path) if path is not None else None - for relative, target in _iter_clamped_source_files(resolved_root): - if scope is not None and target != scope: - continue - text = target.read_text(errors="ignore") - span = _named_span(relative, text, symbol) - if span is None: - continue - lines = text.splitlines() - start, end = span - while start > 1 and lines[start - 2].lstrip().startswith("@"): - start -= 1 # a handler's decorators are part of what the detector must see - body = "\n".join(lines[start - 1 : end]) - return {"path": relative, "name": symbol, "start_line": start, "end_line": end, "text": body[: max(max_chars, 0)]} - return None - - -def entry_points(root: Path, *, path: str | None = None) -> list[dict[str, object]]: - """Handlers the mapper names, with their access classification and evidence. No source text.""" - from .mapping import analyze_entry_points - - resolved_root = root.resolve() - relative = _relative_or_none(resolved_root, path) - targets = _repo_targets(resolved_root) - rows: list[dict[str, object]] = [] - for record in analyze_entry_points(resolved_root, targets): - if relative is not None and record.path != relative: - continue - rows.append( - { - "path": record.path, - "function": record.function_name, - "name": record.name, - "kind": record.kind, - "line": record.line, - "end_line": record.end_line, - "access_level": record.access_level, - "evidence": list(record.access_evidence), - } - ) - return rows - - -def flows(root: Path, *, path: str, function: str) -> list[dict[str, object]]: - """Adjudicated source-to-sink records inside one function: which source, which sink, what the overlay said.""" - from .findings import quick_scan_findings - from .rank import rank_targets - from .semantic import adjudicate - - resolved_root = root.resolve() - relative = _relative_or_none(resolved_root, path) - targets = _repo_targets(resolved_root) - span = _span_in(resolved_root, relative, function) - if span is None: - return [] - findings = [item for item in quick_scan_findings(resolved_root, targets, rank_targets(targets), None) if item.status != "shadow"] - rows: list[dict[str, object]] = [] - for record in adjudicate(root=resolved_root, targets=targets, findings=findings): - if record.path != relative or record.line is None or not span[0] <= record.line <= span[1]: - continue - rows.append( - { - "path": record.path, - "function": function, - "line": record.line, - "sources": list(record.sources), - "sinks": list(record.sinks), - "disposition": record.disposition, - "cwe": record.cwe, - } - ) - return rows - - -def obligations(root: Path, *, path: str, function: str) -> list[dict[str, object]]: - """What one function owes and has not discharged: the operation, the missing discharger, why it is owed.""" - from .config import ObligationsConfig - from .semantic.facts import FactLoadError, load_facts - from .semantic.ir import parse_file - from .semantic.obligations import check_obligations, load_obligation_facts - from .semantic.obligations.dominance import OrderDominance - - resolved_root = root.resolve() - relative = _relative_or_none(resolved_root, path) - targets = _repo_targets(resolved_root) - try: - flow_facts = load_facts() - except FactLoadError: - return [] - irs: dict[str, tuple[object, str]] = {} - texts: dict[str, str] = {} - for target in targets: - try: - text = (resolved_root / target.path).read_text(errors="ignore") - except OSError: - continue - texts[target.path] = text - irs[target.path] = (parse_file(target.path, text, target.language), text) - from .mapping import analyze_entry_points - - result = check_obligations( - irs=irs, # type: ignore[arg-type] - entries=analyze_entry_points(resolved_root, targets), - facts=load_obligation_facts(), - flow_facts=flow_facts, - policy=None, - paths=(), - dominance=OrderDominance(texts=texts), - store_shapes=(), - min_siblings=ObligationsConfig().min_siblings, - ) - return [ - { - "path": finding.operation.path, - "function": finding.operation.function, - "line": finding.operation.line, - "operation": finding.operation.kind, - "resource": finding.operation.resource, - "missing": finding.missing, - "label": finding.label, - "evidence": list(finding.evidence), - } - for finding in result.findings - if (relative is None or finding.operation.path == relative) and finding.operation.function == function - ] - - -def _repo_targets(root: Path) -> list[FileTarget]: - from .preprocess import preprocess_repository - - _, targets = preprocess_repository(root) - return list(targets) - - -def _relative_or_none(root: Path, path: str | None) -> str | None: - if path is None: - return None - return clamp_repo_path(root, path).relative_to(root).as_posix() - - -def _named_span(relative: str, text: str, symbol: str) -> tuple[int, int] | None: - from .preprocess import detect_language - from .semantic.functions import named_function_ranges, spans_named - - ranges = named_function_ranges(relative, text, detect_language(Path(relative))) - spans = spans_named(ranges or (), symbol) - return spans[0] if spans else None - - -def _span_in(root: Path, relative: str | None, function: str) -> tuple[int, int] | None: - if relative is None: - return None - try: - text = (root / relative).read_text(errors="ignore") - except OSError: - return None - return _named_span(relative, text, function) diff --git a/src/openultrasast/learn/features.py b/src/openultrasast/learn/features.py index 8d311cd..aba7352 100644 --- a/src/openultrasast/learn/features.py +++ b/src/openultrasast/learn/features.py @@ -1,10 +1,6 @@ """Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1). -No model. The same per-instrument builders serve two callers: - -- the plane task ``features`` (:mod:`..plane.tasks.features`), which reads a case's delivered artifacts; -- the host: :func:`build_for_scan` over a checkout's quick findings and an engine result, for ``scan`` and - ``pre-push`` (a whole-repository scan, or a base..head delta with :class:`Delta`). +The plane task ``features`` (:mod:`..plane.tasks.features`) reads a case's delivered artifacts. A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the @@ -26,8 +22,8 @@ from typing import Any from ..model.taxonomy import load_families from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files from ..preprocess import build_file_target, detect_language -from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset -from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, infer_for_checkout, origin_of +from ..ruleset import PatternRule +from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, origin_of from .schema import ( BY_NAME, LANGUAGES, @@ -62,12 +58,6 @@ NONE = Part("none") NOT_APPLICABLE = Part("not_applicable") -def language_of(path: str) -> str: - """The closed language of a path (its extension), ``other`` outside :data:`.schema.LANGUAGES`.""" - language = detect_language(Path(path)) - return language if language in LANGUAGES else "other" - - def _clip(name: str, value: Any) -> Any: spec = BY_NAME[name] if value is None or spec.type != "int" or isinstance(value, bool): @@ -328,18 +318,6 @@ def function_of(lines: Sequence[str] | None, path: str, language: str, function: # --- the host path --------------------------------------------------------------------------------------------------- -@dataclass(frozen=True) -class Delta: - """A base..head delta: the base checkout, the head-side changed line ranges per path (``git diff -U0``), and the - base's quick findings and engine result, whose signals decide novelty.""" - - root: Path - changed: Mapping[str, Sequence[tuple[int, int]]] - findings: Sequence[Any] = () - engine_result: Mapping[str, Any] | None = None - pin: str | None = None - - class SourceIndex: """A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level caller graph (the repo-facts patterns) for the entry-point distance.""" @@ -423,12 +401,6 @@ def _family_of(rule: PatternRule | None) -> str: return family.id if family is not None else UNKNOWN_FAMILY -def _engine_state(result: Mapping[str, Any] | None) -> str: - if result is None: - return "none" - return "ran" if int(result.get("questions") or 0) > 0 else "failed" - - def _signals( index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule] ) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]: @@ -447,108 +419,6 @@ def _signals( return quick, engine -def _changed_in(index: SourceIndex, changed: Mapping[str, Sequence[tuple[int, int]]]) -> dict[tuple[str, str], int]: - """(path, function) -> changed lines inside it, by the head's enclosing declarations.""" - counts: dict[tuple[str, str], int] = {} - for path, ranges in changed.items(): - for lo, hi in ranges: - for line in range(int(lo), int(hi) + 1): - key = (path, index.function_at(path, line)) - counts[key] = counts.get(key, 0) + 1 - return counts - - -def _body_calls(lines: Sequence[str] | None, language: str, function: str) -> set[str]: - if not lines: - return set() - return {m.group(1) for i, line in enumerate(lines) if enclosing(lines, i, language) == function for m in _CALL.finditer(line)} - - -def build_for_scan( - root: Path, - findings: Sequence[Any], - engine_result: Mapping[str, Any] | None, - *, - base: Delta | None = None, - rules: Sequence[PatternRule] | None = None, - quick_languages: Iterable[str] | None = None, - engine_languages: Iterable[str] | None = None, - roles: RoleSet | None | bool = True, -) -> list[dict[str, Any]]: - """``static`` records for a checkout: one per (path, function, family) with a quick or engine signal. ``findings`` - are the quick findings (enabled and shadow alike), ``engine_result`` the engine's record (``findings`` with - ``site``/``family``/``rung``/``witness``, ``questions``, ``completed``, ``degradations``; ``None`` when the engine - did not run, ``questions == 0`` is ``failed``). With ``base`` only functions enclosing a changed line are - candidates, and ``delta.*`` compares their signals with the base's. ``roles``: ``True`` infers wrapper roles over - the checkout (priors off), a :class:`.roles.RoleSet` is used as given, ``False``/``None`` leaves ``roles`` ``none``.""" - from ..plane.tasks.alerts import covers - from ..plane.tasks.alerts import engine_languages as engine_covered - from ..plane.tasks.alerts import quick_languages as quick_covered - from ..semantic.facts import FactLoadError, load_facts - - root = Path(root) - loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR) - by_id = {rule.rule_id: rule for rule in loaded} - quick_langs = set(quick_languages if quick_languages is not None else quick_covered()) - engine_langs = set(engine_languages if engine_languages is not None else engine_covered()) - index = SourceIndex(root) - quick, engine = _signals(index, findings, engine_result, by_id) - keys = sorted(set(quick) | set(engine)) - changed: dict[tuple[str, str], int] = {} - base_quick: dict[tuple[str, str, str], list[tuple[str, str]]] = {} - base_engine: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {} - base_index: SourceIndex | None = None - if base is not None: - changed = _changed_in(index, base.changed) - keys = [k for k in keys if (k[0], k[1]) in changed] - base_index = SourceIndex(base.root) - base_quick, base_engine = _signals(base_index, base.findings, base.engine_result, by_id) - entries, file_entries = entry_names(root, {k[0] for k in keys}) if keys else (set(), set()) - callers = index.callers() if keys else {} - quick_version = ruleset_digest(loaded) - engine_state = _engine_state(engine_result) - completed, questions = (engine_result or {}).get("completed"), (engine_result or {}).get("questions") - completion = int(completed) / int(questions) if isinstance(completed, int) and isinstance(questions, int) and questions else None - degraded = bool((engine_result or {}).get("degradations")) - vocabularies: dict[str, Vocabulary] = {} - inferred = (infer_for_checkout(root) if keys else RoleSet(sources=("inferred_wrapper",))) if roles is True else roles or None - try: - sanitizer_calls = {language: {c for s in load_facts().for_language(language).sanitizers for c in s.calls} for language in LANGUAGES} - except FactLoadError: - sanitizer_calls = {} - out: list[dict[str, Any]] = [] - for key in keys: - path, function, family = key - language = detect_language(Path(path)) - parts: dict[str, Part] = {} - parts["quick"] = quick_part(quick.get(key, []), by_id, quick_version) if covers(quick_langs, language) else NONE - if language in engine_langs and engine_state != "none": - vocabulary = vocabularies.setdefault(language, Vocabulary.load(language)) - parts["engine"] = ( - engine_part(engine.get(key, []), completion=completion, degraded=degraded, vocabulary=vocabulary) - if engine_state == "ran" else Part("failed") - ) # fmt: skip - lines = index.lines(path) - parts["source"] = source_part(lines, language, function) - parts["roles"] = roles_part(function_of(lines, path, language, function), inferred, language) - distance = 0 if function == GLOBAL and path in file_entries else entry_distance(function, entries, callers) - parts["entry_points"] = Part("ran", {"facts.entry_distance": distance}) - parts["facts"] = facts_part(_callers_of(index, callers, path, function), function) - if base is None or base_index is None: - parts["delta"] = NOT_APPLICABLE - else: - head_n = len(quick.get(key, [])) + len(engine.get(key, [])) - base_n = len(base_quick.get(key, [])) + len(base_engine.get(key, [])) - novelty = "new" if base_n == 0 else "worsened" if head_n > base_n else "unchanged" - known = sanitizer_calls.get(language, set()) - removed = bool((_body_calls(base_index.lines(path), language, function) - _body_calls(lines, language, function)) & known) - values = {"delta.novelty": novelty, "delta.changed_lines": changed.get((path, function), 0), "delta.sanitizer_removed": removed} - parts["delta"] = Part("ran", values) - unit, pin = ("delta", base.pin) if base is not None else ("pin", None) - out.append(record(f"{path}::{function}", family, language, "static", parts, unit=unit, base=pin)) - return out - - def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]: """The call sites of ``function`` in other product files, as repo-facts records them (capped at 40).""" if function == GLOBAL or function not in callers: @@ -567,8 +437,8 @@ def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, __all__ = [ - "CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Delta", "Part", "SourceIndex", "Vocabulary", "agree_part", + "CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Part", "SourceIndex", "Vocabulary", "agree_part", "function_of", "model_sinks_part", "roles_part", - "build_for_scan", "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "language_of", "quick_part", "record", + "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "quick_part", "record", "ruleset_digest", "source_part", "verify_part", ] # fmt: skip diff --git a/src/openultrasast/learn/folds.py b/src/openultrasast/learn/folds.py index c278c33..1c7e293 100644 --- a/src/openultrasast/learn/folds.py +++ b/src/openultrasast/learn/folds.py @@ -38,7 +38,7 @@ class FoldError(RuntimeError): class Fold: """One evaluation (or compile, or deployment) boundary: the groups evaluated, and what retrieval must not use.""" - kind: str # outer | leave_one_source_out | leave_one_framework_out | compile | deployment + kind: str # outer | compile | deployment name: str eval_groups: frozenset[str] held_source: str | None = None @@ -126,37 +126,6 @@ def outer_folds(examples: Sequence[Example], *, k: int = OUTER_K, seed: int = 0, return [Fold("outer", f"outer-{i}", frozenset(groups)) for i, groups in enumerate(assigned)] -def leave_one_source_out(examples: Sequence[Example], *, exclude: Iterable[str] = ()) -> list[Fold]: - excluded = set(exclude) - by_source: dict[str, set[str]] = {} - for example in examples: - if example.group not in excluded: - by_source.setdefault(example.source, set()).add(example.group) - return [Fold("leave_one_source_out", f"source-{s}", frozenset(g), held_source=s) for s, g in sorted(by_source.items())] - - -def leave_one_framework_out( - examples: Sequence[Example], *, exclude: Iterable[str] = (), min_groups: int = FRAMEWORK_MIN_GROUPS -) -> list[Fold]: - """One fold per framework seen; below ``min_groups`` groups it is ``insufficient data`` and evaluates nothing.""" - excluded = set(exclude) - by_framework: dict[str, set[str]] = {} - for example in examples: - for framework in example.frameworks: - if example.group not in excluded: - by_framework.setdefault(framework, set()).add(example.group) - folds = [] - for framework, groups in sorted(by_framework.items()): - enough = len(groups) >= min_groups - folds.append( - Fold( - "leave_one_framework_out", f"framework-{framework}", frozenset(groups) if enough else frozenset(), - held_framework=framework, status="ok" if enough else INSUFFICIENT, - ) - ) # fmt: skip - return folds - - def assert_disjoint(split: CompileSplit, folds: Iterable[Fold]) -> None: """The compile split shares no group with any evaluation fold; outer folds are pairwise disjoint.""" seen: dict[str, str] = {} @@ -181,6 +150,5 @@ def folds_digest(split: CompileSplit, folds: Iterable[Fold]) -> str: __all__ = [ "COMPILE_FRACTION", "FRAMEWORK_MIN_GROUPS", "INSUFFICIENT", "OUTER_K", "CompileSplit", "Fold", "FoldError", "GroupInfo", - "assert_disjoint", "compile_split", "folds_digest", "group_table", "leave_one_framework_out", "leave_one_source_out", - "outer_folds", + "assert_disjoint", "compile_split", "folds_digest", "group_table", "outer_folds", ] # fmt: skip diff --git a/src/openultrasast/learn/roles.py b/src/openultrasast/learn/roles.py index 04f18c7..0229988 100644 --- a/src/openultrasast/learn/roles.py +++ b/src/openultrasast/learn/roles.py @@ -23,7 +23,6 @@ finding. A role's ``path``/``function`` are provenance, never features. from __future__ import annotations -import json import re from collections import Counter from collections.abc import Iterable, Mapping, Sequence @@ -536,13 +535,9 @@ def _confidence(role: Role) -> float: return round(1.0 / (role.depth + 1), 6) -def load_roles(path: Path) -> RoleSet: - return RoleSet.from_json(json.loads(Path(path).read_text(encoding="utf-8"))) - - __all__ = [ "CWE_OPERATION", "DEPENDENCY_DIRS", "MAX_DEPTH", "MODEL_ROLE_KINDS", "OPERATION_CWE", "ROLE_KINDS", "SOURCE_KIND", "VERSION", "Function", "Role", "RoleSet", "base_roles", "body_roles", "dependency_files", "from_model_roles", "functions_of", - "infer_for_checkout", "infer_wrappers", "language_key", "load_roles", "mentions", "origin_of", "parameters", + "infer_for_checkout", "infer_wrappers", "language_key", "mentions", "origin_of", "parameters", "vocabulary_overlay", ] # fmt: skip diff --git a/src/openultrasast/mapping.py b/src/openultrasast/mapping.py index 56d566e..b652378 100644 --- a/src/openultrasast/mapping.py +++ b/src/openultrasast/mapping.py @@ -35,31 +35,6 @@ class StaticHint: evidence_candidate: bool = True -@dataclass(frozen=True) -class MappingTaskRecord: - kind: MappingKind - path: str - rule_id: str - task: str - rationale: str - - -@dataclass(frozen=True) -class DifferentialMappingRecord: - path: str - change_kind: str - trust_boundary: str | None - rationale: str - - -@dataclass(frozen=True) -class SharpEdgeRecord: - path: str - category: str - surface: str - rationale: str - - @dataclass(frozen=True) class EntryPointRecord: path: str @@ -185,58 +160,6 @@ def attach_reachability_hints(targets: list[FileTarget], entry_points: list[Entr return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets] -def verifier_evidence_candidates(hints: list[StaticHint], *, path: str | None = None) -> list[dict[str, object]]: - selected = [hint for hint in hints if hint.evidence_candidate and (path is None or hint.path == path)] - return [ - { - "source": hint.analyzer, - "rule_id": hint.rule_id, - "path": hint.path, - "line": hint.start_line, - "severity": hint.severity, - "message": hint.message, - "provenance": hint.provenance, - } - for hint in selected - ] - - -def semgrep_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]: - return [ - MappingTaskRecord( - kind="semgrep", - path=hint.path, - rule_id=hint.rule_id, - task="pattern_variant_review", - rationale="Use Semgrep evidence to check variants and rule precision before verifier promotion.", - ) - for hint in hints - if hint.analyzer == "semgrep" - ] - - -def codeql_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]: - return [ - MappingTaskRecord( - kind="codeql", - path=hint.path, - rule_id=hint.rule_id, - task="source_sink_sanitizer_path_review", - rationale="Use CodeQL evidence to inspect source, sink, sanitizer, and path explanation.", - ) - for hint in hints - if hint.analyzer == "codeql" - ] - - -def differential_mapping_record(path: str, *, change_kind: str, trust_boundary: str | None, rationale: str) -> DifferentialMappingRecord: - return DifferentialMappingRecord(path=path, change_kind=change_kind, trust_boundary=trust_boundary, rationale=rationale) - - -def sharp_edge_record(path: str, *, category: str, surface: str, rationale: str) -> SharpEdgeRecord: - return SharpEdgeRecord(path=path, category=category, surface=surface, rationale=rationale) - - def write_static_hints(hints: list[StaticHint], path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n") diff --git a/src/openultrasast/model/config_value.py b/src/openultrasast/model/config_value.py index 0b0862f..b2811ff 100644 --- a/src/openultrasast/model/config_value.py +++ b/src/openultrasast/model/config_value.py @@ -87,12 +87,6 @@ def verdicts(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str return [] -def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str = "") -> Verdict | None: - """The strongest single verdict, kept for callers that ask one question and want one answer.""" - answers = verdicts(cpg, spec, function=function, file=file) - return answers[0] if answers else None - - def _location(row: Mapping[str, object], file: str) -> str: """``path:line:function`` of the setting, when the region named a file. @@ -156,4 +150,4 @@ def _settings(rows: object, spec: ConfigSpec, *, function: str) -> list[dict[str return sorted(kept, key=lambda row: (str(row["method"]), _line(row), str(row["setting"]))) -__all__ = ["request_params", "verdict", "verdicts"] +__all__ = ["request_params", "verdicts"] diff --git a/src/openultrasast/model/pipeline.py b/src/openultrasast/model/pipeline.py index 93e38a7..70cedb5 100644 --- a/src/openultrasast/model/pipeline.py +++ b/src/openultrasast/model/pipeline.py @@ -29,12 +29,10 @@ from typing import Any from ..cpg.backend import CpgResult from ..redaction import redact_secrets -from .config_value import verdict as config_verdict from .config_value import verdicts as config_verdicts from .dominance import verdict as dominance_verdict from .ladder import Rung, Verdict from .specs import ConfigSpec, DominanceSpec, TaintSpec -from .taint import verdict as taint_verdict from .taint import verdicts as taint_verdicts logger = logging.getLogger(__name__) @@ -93,34 +91,11 @@ def _arbitrate_all( if isinstance(spec, ConfigSpec): return config_verdicts(cpg, spec, function=function, file=path) if isinstance(spec, DominanceSpec): - answer = _arbitrate(cpg, spec, path=path, function=function, parameter_sources=parameter_sources, call_depth=call_depth) + answer = dominance_verdict(cpg, spec, function=function, file=path) return [answer] if answer is not None else [] return taint_verdicts(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth) -def _arbitrate( - cpg: CpgResult, - spec: TaintSpec | DominanceSpec | ConfigSpec, - *, - path: str, - function: str, - parameter_sources: bool, - call_depth: int, -) -> Verdict | None: - """Route a family to the arbiter that can actually decide it. - - Dispatching on the spec type rather than on a family name keeps this honest: a family without an arbiter - cannot be silently routed to the wrong one. The pipeline first shipped taint-only, so access_control and - config_secrets -- whose arbiters had worked since group 4 -- produced nothing, and every one of their - pairs read as `both_silent` on the first end-to-end run. - """ - if isinstance(spec, DominanceSpec): - return dominance_verdict(cpg, spec, function=function, file=path) - if isinstance(spec, ConfigSpec): - return config_verdict(cpg, spec, function=function, file=path) - return taint_verdict(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth) - - def scan_region( cpg: CpgResult, spec: TaintSpec | DominanceSpec | ConfigSpec, diff --git a/src/openultrasast/model/report.py b/src/openultrasast/model/report.py index 6dbf8d1..6443efa 100644 --- a/src/openultrasast/model/report.py +++ b/src/openultrasast/model/report.py @@ -1,87 +1,9 @@ -"""Reporting that cannot flatter itself (model-grounded-detection, Req 9.3, 11.1). - -Three disciplines survive from the deleted publish/scoring layer, and each is here because its absence caused -a real error in this project's history rather than because it is tidy: - -* **Per-slice rows, always.** A headline over mixed slices once hid that agent-written code scored 44.4% - while the hand-written development slice scored 83.3%. `render` refuses to produce anything without rows. -* **The overfitting gap beside the headline.** The distance between the development slice and the deciding - real-world one is stated, never left for the reader to compute. When a slice is missing the gap is - *absent*, not zero — reading a missing measurement as "no overfitting" is the flattering failure. -* **The rung on every finding.** A finding's strength is what established it; omitting the rung invites a - reader to treat a suspicion as a result. -""" - +"""Report model coverage for the scanned regions.""" from __future__ import annotations from collections.abc import Sequence -from dataclasses import dataclass -from ..findings import StaticFinding - - -@dataclass(frozen=True) -class SliceRow: - """One slice's coverage. `covered` is pairs the model *split*, not pairs it flagged.""" - - slice: str - pairs: int - covered: int - - @property - def rate(self) -> float: - return self.covered / self.pairs if self.pairs else 0.0 - - -def overfitting_gap(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> float | None: - """Development-slice coverage minus the deciding slice's, or ``None`` when either is absent. - - ``None`` rather than ``0.0`` on purpose: a slice that was never measured must not read as agreement - between them. - """ - by_name = {row.slice: row for row in rows} - if development not in by_name or deciding not in by_name: - return None - return by_name[development].rate - by_name[deciding].rate - - -def render(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> str: - """The report. Raises when there are no per-slice rows, because a bare aggregate is the thing to prevent.""" - if not rows: - raise ValueError("a report needs per-slice rows: there is no headline without them") - - lines = ["## Model coverage", "", "| slice | pairs | covered | coverage |", "|---|---|---|---|"] - for row in sorted(rows, key=lambda r: r.slice): - lines.append(f"| {row.slice} | {row.pairs} | {row.covered} | {row.rate:.0%} |") - - total_pairs = sum(row.pairs for row in rows) - total_covered = sum(row.covered for row in rows) - lines += [ - "", - f"Across every slice: {total_covered}/{total_pairs} = {total_covered / total_pairs:.0%} — read the rows above it, not this line.", - ] - - gap = overfitting_gap(rows, development=development, deciding=deciding) - if gap is not None: - lines += [ - "", - f"**Overfitting gap: {gap:+.0%}** — `{development}` (development) minus `{deciding}`, which is the deciding real-world slice.", - ] - elif development or deciding: - lines += ["", f"**Overfitting gap: not measurable** — `{development}` or `{deciding}` is absent from this run."] - return "\n".join(lines) + "\n" - - -def render_findings(findings: Sequence[StaticFinding]) -> str: - """Findings with the rung that established each one (Req 5.3).""" - lines = ["## Findings", "", "| finding | location | rung | severity |", "|---|---|---|---|"] - for finding in findings: - where = f"{finding.path}:{finding.line}" if finding.line else finding.path - lines.append(f"| {finding.title} | {where} | `{finding.rung}` | {finding.severity} |") - return "\n".join(lines) + "\n" - - -__all__ = ["SliceRow", "overfitting_gap", "render", "render_findings"] +__all__ = [] def coverage_rows(regions: Sequence[object]) -> list[dict[str, object]]: diff --git a/src/openultrasast/model/taint.py b/src/openultrasast/model/taint.py index b942f74..0460437 100644 --- a/src/openultrasast/model/taint.py +++ b/src/openultrasast/model/taint.py @@ -5,7 +5,7 @@ graph establishes it and the LLM's claim is no longer load-bearing. A path a san `CORROBORATED`: the flow is real, but whether the sanitizer is *sufficient* is a semantic judgement the model cannot make, and that residual is the LLM's — checked against this verdict rather than averaged over runs. -No path at all yields `None`, not a negative verdict: the caller reports `suspicion`. And a query that failed +No path at all yields an empty list, not a negative verdict: the caller reports `suspicion`. And a query that failed yields `None` too, because "the engine could not decide" is not "there is no flow" — conflating them is how a missing engine would silently become a clean bill of health. """ @@ -69,37 +69,6 @@ def request_params( } -def verdict( - cpg: CpgResult, spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False, call_depth: int = 0 -) -> Verdict | None: - """The strongest verdict the taint query supports for ``spec`` in ``function``, or ``None`` to stay at suspicion. - - ``parameter_sources`` additionally treats the labeled function's own parameters as untrusted. That is right - for a function-level pair, where the function boundary *is* the trust boundary, and wrong for a whole - repository, where most parameters carry internal values — so it is off by default and the caller opts in. - An ENTRY POINT is the case where both are true at once: its parameters are attacker input by definition, - which is what the driver uses to decide. - - ``call_depth`` admits sinks that many call-graph levels below the labeled method. Zero keeps the old - function-local question; above zero is what lets a handler's parameter reach a sink in another module, - which is the shape VAmPI's SQL injection has and the shape a function-scoped question cannot see. - """ - # The strongest of the per-site verdicts, which is what this always meant. It used to run its own - # selection over the union of flows, and two copies of "which flow decides" is the duplication this - # module has been bitten by before: the per-site preference added to `verdicts` did not reach here, so - # the two entry points disagreed about the same rows. - # - # A flow the graph cannot see a discharge for is the finding; a sanitized or BOUNDED one is a question. - # libpng bounds `strcpy(outname+len, ".png")` with `(len = strlen(inname)) > 250` against a char[256], - # and entailing it is a claim the code contradicts. But the bound reaches the sink through an `error` - # flag rather than by dominating it, so the graph has not shown the copy is guarded either -- and a - # bound is no proof of safety in any case, off-by-one being the classic way they fail. Corroborated is - # the honest rung: the model cannot assert this, so the judge is asked instead of the site being - # silenced. - answers = verdicts(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth) - return answers[0] if answers else None - - def _usable_flows( cpg: CpgResult, spec: TaintSpec, @@ -137,17 +106,7 @@ def verdicts( parameter_sources: bool = False, call_depth: int = 0, ) -> list[Verdict]: - """One verdict per distinct SINK SITE, strongest first. - - ``verdict`` answers "what is the strongest thing to say about this region", which is the right question - for a pair: one labelled function, one bug, one answer. It is the wrong question for a region that spans - a file. A PHP file with an SQL injection on line 6 and a command injection on line 13 got ONE injection - verdict, and `system` won it on flow length -- so the SQL injection went unreported, by construction - rather than by any failure of the analysis. - - The first element is exactly what ``verdict`` returns, so the pair path and the committed measurements - that rest on it are unchanged. - """ + """One verdict per distinct sink site, strongest first.""" flows = _usable_flows(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth) if not flows: return [] @@ -164,7 +123,7 @@ def verdicts( preferred = [flow for flow in site_flows if _from_modelled_source(flow, spec)] or list(site_flows) open_flows = [flow for flow in preferred if not flow["sanitized"] and not flow["bounded"]] chosen.append(min(open_flows or preferred, key=lambda flow: _rank(flow, spec))) - # Same total order the single-verdict path uses, so `verdicts(...)[0] == verdict(...)`. + # Sort strongest first, then by site for stable results. chosen.sort(key=lambda flow: (bool(flow["sanitized"]) or bool(flow["bounded"]), _rank(flow, spec))) return [_verdict_for(flow, spec) for flow in chosen] @@ -309,4 +268,4 @@ def _location(flow: Mapping[str, object]) -> str: return f"{where}:{line}:{flow.get('sinkMethod') or '?'}" -__all__ = ["request_params", "verdict"] +__all__ = ["request_params", "verdicts"] diff --git a/src/openultrasast/model/taxonomy.py b/src/openultrasast/model/taxonomy.py index 0c87bce..c5350fa 100644 --- a/src/openultrasast/model/taxonomy.py +++ b/src/openultrasast/model/taxonomy.py @@ -13,7 +13,6 @@ from __future__ import annotations import tomllib from dataclasses import dataclass from pathlib import Path -from typing import Literal FAMILY_IDS = ( "injection", @@ -28,7 +27,6 @@ FAMILY_IDS = ( "unknown", ) VERIFIER_KINDS = ("canary", "static", "none") -Relation = Literal["same", "parent_child", "lateral", "fabricated"] DEFAULT_FAMILIES_PATH = Path(__file__).resolve().parents[1] / "ruleset" / "families.toml" _FAMILY_FIELDS = frozenset({"id", "description", "cwes", "mechanisms", "verifier", "parent", "limits"}) @@ -72,24 +70,6 @@ class FamilyTaxonomy: def family_of_mechanism(self, mechanism: str) -> Family | None: return next((family for family in self.families if mechanism in family.mechanisms), None) - @property - def hierarchical(self) -> bool: - """True when some family declares a parent. While this is False, partial credit can never be earned.""" - return any(family.parent for family in self.families) - - def related(self, left: str, right: str) -> Relation: - """How two family answers relate. Abstention is not partial credit: ``unknown`` against a real - family is ``lateral``, never ``parent_child``.""" - known = {family.id: family for family in self.families} - if left not in known or right not in known: - return "fabricated" - if left == right: - return "same" - if known[left].parent == right or known[right].parent == left: - return "parent_child" - return "lateral" - - def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy: """Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file); a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``.""" @@ -189,7 +169,6 @@ __all__ = [ "FamiliesError", "Family", "FamilyTaxonomy", - "Relation", "load_families", ] diff --git a/src/openultrasast/plane/delivery.py b/src/openultrasast/plane/delivery.py index 420dcf3..bb60739 100644 --- a/src/openultrasast/plane/delivery.py +++ b/src/openultrasast/plane/delivery.py @@ -13,7 +13,6 @@ Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``):: runs///output.tar the runner's PUT runs/// each declared output, re-put by the reconciler runs///done.json status, bytes, file count, finished - runs//inputs/ an input no task produces (``put_input``) Nothing here logs or prints a URL. """ @@ -164,11 +163,6 @@ class Delivery: put.append(artifact) return put - def put_input(self, name: str, path: Path) -> str: - """Store a file no task produces as ``runs//inputs/``; returns its presigned GET URL.""" - self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes()) - return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires) - # --- state --------------------------------------------------------------------------------------------------- def save_state(self, data: Mapping[str, Any]) -> None: diff --git a/src/openultrasast/plane/egress.py b/src/openultrasast/plane/egress.py index 57a487f..09cc4a1 100644 --- a/src/openultrasast/plane/egress.py +++ b/src/openultrasast/plane/egress.py @@ -1,61 +1,29 @@ -"""Per-task network egress on Agent Substrate (ai-service-plane, task 0: gateway + receiver Service). +"""Per-task TLS egress to the store, Git hosts and model endpoints. -Every TCP connection an actor opens (DNS aside) is redirected to Substrate's egress gateway -(``atenet-egress.ate-system``), which denies by default: an actor without an ``EgressPolicy`` gets no tunnel, and -a refused connection is accepted and then closed. :func:`policy_for` grants a task exactly what it needs: - -* ``http`` on port 80 to the artifact receiver's cluster name (``ousast-receiver.ax-system.svc.cluster.local``, a - selector-less Service whose endpoint is this host's receiver, ``ops/ax/receiver-service.yaml.tmpl``). The actor - cannot resolve cluster names, so the runner dials ``OUSAST_ARTIFACT_DIAL`` (the Service's ClusterIP, port 80) - with the receiver's name as ``Host``. The gateway decides the policy on that ``Host`` but connects to the - address the actor dialled (measured live: agentgateway's upstream is the CONNECT authority, not the Host), so - the dial address must be one the gateway pod reaches: the ClusterIP, which kube-proxy maps to the host. -* ``tls_passthrough`` on 443 for the Git hosts of the bound Workspaces' ``https`` repos (smart HTTP talks to the - repo's host only) and for the bound Model's ``openultrasast.io/egress-hosts`` (``api.deepseek.com``). - -Nothing else: no wildcard, no IP address (the gateway rejects them), no ``https`` rule (that one intercepts TLS and -the actor would have to trust the gateway CA, which ax templates do not mount). :class:`Egress` writes the policy -with ``kubectl ate create egress-policy`` after ``ax apply`` created the actor, reads it back, and before resume -waits ``OUSAST_EGRESS_SETTLE`` s (11) because the gateway caches a missing policy as deny for 10 s. The kubectl-ate -CLI has no delete verb: Substrate's store deletes the policy with its actor (``ON DELETE CASCADE``), so -:meth:`Egress.delete` runs after ``ax delete task`` and confirms the policy is gone. +The gateway denies by default. Egress writes and checks each actor's policy before resume, +then waits for the cached denial to expire. Deleting the actor removes its policy. """ from __future__ import annotations -import io import ipaddress import json import os -import shutil import subprocess -import tarfile -import threading import time from collections.abc import Iterable, Mapping -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path from typing import Any -from urllib.parse import unquote, urlsplit +from urllib.parse import urlsplit from .manifests import Model, Task, Workspace __all__ = [ - "INPUTS_PATH", - "RECEIVER_HOST", - "RECEIVER_PORT", "Egress", - "Receiver", "git_hosts", - "receiver_cluster_ip", "policy_for", - "receiver_address", "rules_of", ] -RECEIVER_HOST = "ousast-receiver.ax-system.svc.cluster.local" -INPUTS_PATH = "/inputs/" # GET /inputs//: a consumer fetches a declared input -RECEIVER_PORT = 18090 # the receiver Service's targetPort; ops/ax/up.sh renders the Service with the same number _NOT_FOUND = ("not found", "notfound") @@ -89,12 +57,10 @@ def policy_for( workspaces: Iterable[Workspace], model: Model | None, store_host: str | None, - receiver_host: str | None = None, ) -> dict[str, Any]: """The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the - bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless - ``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under + bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. An http Git repo cannot be reached under this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows hostnames only).""" tls, plain = git_hosts(workspaces) @@ -106,8 +72,6 @@ def policy_for( raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname") hosts.add(store_host.lower()) rules: list[dict[str, Any]] = [] - if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it) - rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}}) if hosts: rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}}) return {"rules": rules} @@ -125,41 +89,6 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]: return sorted(out, key=lambda r: json.dumps(r, sort_keys=True)) -def receiver_cluster_ip(context: str) -> str | None: - """The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when - kubectl cannot say.""" - if not context: - raise RuntimeError("receiver_cluster_ip needs the profile's kube context") - argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"] - try: - proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20) - except (OSError, subprocess.TimeoutExpired): - return None - address = proc.stdout.strip() - return address if proc.returncode == 0 and _is_ip(address) else None - - -def receiver_address(port: int, context: str | None) -> tuple[str, str | None]: - """(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the - profile's kube context the Service's ClusterIP is looked up in. - - ``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://:/`` and no dial (tests, a host the actor - reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the - Service's ClusterIP on port 80: the gateway checks the policy against ``Host`` but connects to the address the - actor dialled (its CONNECT authority), so that address must reach the receiver from the gateway pod. - """ - host = os.environ.get("OUSAST_ARTIFACT_HOST") - if host: - return f"http://{host}:{port}/", None - dial = os.environ.get("OUSAST_ARTIFACT_DIAL") - if not dial: - if not context: - raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up") - address = receiver_cluster_ip(context) - dial = f"{address}:80" if address else None - return f"http://{RECEIVER_HOST}/", dial - - class Egress: """``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake.""" @@ -230,95 +159,3 @@ class Egress: time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11)) -class Receiver(ThreadingHTTPServer): - """Accepts one tar per running task and extracts it under ``//`` (POST); serves a running task - the producer artifacts it declared as inputs (``GET /inputs//``, ``inputs[task]``).""" - - daemon_threads = True - - def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None: - super().__init__(("0.0.0.0", port), _Handler) - self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]() - self.inputs: dict[str, frozenset[str]] = {} - self.lock = threading.Lock() - self.url, self.dial = receiver_address(self.server_address[1], context) - - def _task(self, headers: Any) -> str | None: - """The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None.""" - task = str(headers.get("X-Ousast-Task", "")) - with self.lock: - return task if headers.get("X-Ousast-Run") == self.run_name and task in self.running else None - - def input_file(self, headers: Any, ref: str) -> tuple[int, str | Path]: - """(200, the file) when ``ref`` is one of the requesting task's declared inputs, else (403/404, why). - - ``ref`` must equal a declared ``/`` exactly and resolve to a regular file under the run - directory; nothing is normalised first, so ``..`` or an absolute path never names a declared input.""" - task = self._task(headers) - if task is None: - return 403, "unknown run or task" - with self.lock: - declared = ref in self.inputs.get(task, frozenset()) - if not declared: - return 403, f"{ref!r} is not an input of {task}" - root = self.base.resolve() - path = (root / ref).resolve() - if not path.is_relative_to(root) or not path.is_file(): - return 404, f"{ref} was not delivered" - return 200, path - - def accept(self, headers: Any, body: bytes) -> tuple[int, str]: - task = self._task(headers) - if task is None: - return 403, "unknown run or task" - if not str(headers.get("Content-Type", "")).startswith("application/x-tar"): - return 415, "expected application/x-tar" - dest = (self.base / task).resolve() - try: - with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as tar: - members = tar.getmembers() - for m in members: - if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest): - return 400, f"rejected member {m.name}" - dest.mkdir(parents=True, exist_ok=True) - tar.extractall(dest, members) # every member checked above - except tarfile.TarError as exc: - return 400, f"bad tar: {exc}" - with self.lock: - self.delivered.add(task) - return 200, "ok" - - -class _Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: # noqa: N802 - http.server's naming - server: Receiver = self.server # type: ignore[assignment] - path = urlsplit(self.path).path - if not path.startswith(INPUTS_PATH): - self._text(404, "not found") - return - code, found = server.input_file(self.headers, unquote(path[len(INPUTS_PATH) :])) - if not isinstance(found, Path): - self._text(code, found) - return - with found.open("rb") as source: # streamed: a producer artifact can be far larger than any env value - self.send_response(200) - self.send_header("Content-Type", "application/octet-stream") - self.send_header("Content-Length", str(os.fstat(source.fileno()).st_size)) - self.end_headers() - shutil.copyfileobj(source, self.wfile, 1 << 16) - - def do_POST(self) -> None: # noqa: N802 - http.server's naming - server: Receiver = self.server # type: ignore[assignment] - body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) - self._text(*server.accept(self.headers, body)) - - def _text(self, code: int, text: str) -> None: - data = text.encode() - self.send_response(code) - self.send_header("Content-Type", "text/plain") - self.send_header("Content-Length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: object) -> None: - return diff --git a/src/openultrasast/plane/reconciler.py b/src/openultrasast/plane/reconciler.py index 279d893..6b367f9 100644 --- a/src/openultrasast/plane/reconciler.py +++ b/src/openultrasast/plane/reconciler.py @@ -10,8 +10,7 @@ credential from this process's environment, never from a file (Req 4.6), and the ``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without -delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind -profile for group 2 only (task 2.6 removes it). +delivery fail it; ``ax delete task`` follows. ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as @@ -40,13 +39,12 @@ from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path from typing import Any -from urllib.parse import urlsplit import yaml from .delivery import Delivery, DeliveryError, declared from .doctor import doctor -from .egress import RECEIVER_PORT, Egress, Receiver, policy_for +from .egress import Egress, policy_for from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests from .memory import open_store from .profile import PlaneProfile, load_profile @@ -142,16 +140,14 @@ def _model_doc(model: Model) -> dict[str, Any]: return _doc("Model", model.metadata.name, spec, model.metadata.atespace) -def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]: +def render_task(run: Run, entry: RunTask, manifests: Manifests) -> list[dict[str, Any]]: """The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them - (``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given.""" + (``OUSAST_INPUTS``) after the start by presigned URL.""" task: Task = manifests.tasks[entry.task] ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT) env = {e.name: e.value for e in task.env} env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}") - if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env - env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {})) budget = _fields(entry.budget) if entry.budget else {} env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()}) inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()} @@ -341,25 +337,22 @@ def _execute( base: Path, delivery: Delivery, router: Router, - receiver: Receiver | None = None, ) -> tuple[str, str]: ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task] atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml" manifest.parent.mkdir(parents=True, exist_ok=True) - docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None) + docs = render_task(run, entry, manifests) manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8") model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "") workspaces = [manifests.workspaces[b.name] for b in task.workspaces] - body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body - store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None) - if receiver is None: - delivery.reset(entry.name) # a previous attempt's tar must not complete this one + body = delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body + delivery.reset(entry.name) # a previous attempt's tar must not complete this one def delivered() -> bool: - return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name) + return delivery.delivered(entry.name) try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request - policy = policy_for(task, workspaces, model, store_host, receiver_host) + policy = policy_for(task, workspaces, model, delivery.host) start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body) ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists egress.apply(ax_name, atespace, policy) @@ -369,7 +362,7 @@ def _execute( return "failed", str(exc) try: failure = _await(ax, ax_name, delivered, start) - if failure is None and receiver is None: + if failure is None: delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared except DeliveryError as exc: failure = str(exc) @@ -408,11 +401,6 @@ def run( _read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}}, mirror=delivery.save_state, ) - receiver: Receiver | None = None - if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it) - port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT) - receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT - threading.Thread(target=receiver.serve_forever, daemon=True).start() cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]() pending = [t for t in run_spec.tasks if state.status(t.name) != "done"] for entry in run_spec.tasks: @@ -427,7 +415,7 @@ def run( cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it state.set(entry.name, status="running", started=_now(), finished=None) try: - outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver) + outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router) except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue outcome = ("failed", f"{type(exc).__name__}: {exc}") finished.put((entry.name, *outcome)) @@ -444,27 +432,17 @@ def run( pending.remove(entry) running[entry.name] = entry blocked.add(entry.serialize or "") - if receiver is not None: - with receiver.lock: # the receiver serves a running task its declared inputs, nothing else - receiver.running.add(entry.name) - receiver.inputs[entry.name] = frozenset(entry.inputs.values()) threading.Thread(target=worker, args=(entry,), daemon=True).start() if not running: break # nothing can start: a producer stopped short name, status, reason = finished.get() del running[name] - if receiver is not None: - with receiver.lock: - receiver.running.discard(name) - receiver.inputs.pop(name, None) summary = _read_json(base / name / "summary.json") or {} state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None) halted = halted or status == "failed" if status == "unfinished": pending = [t for t in pending if name not in t.producers] finally: - if receiver is not None: - receiver.shutdown() lock.unlink(missing_ok=True) statuses = [state.status(t.name) for t in run_spec.tasks] result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished" @@ -546,4 +524,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No return "\n".join(lines) -__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"] +__all__ = ["Ax", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"] diff --git a/src/openultrasast/plane/runner.py b/src/openultrasast/plane/runner.py index 631dba8..2b1c23b 100644 --- a/src/openultrasast/plane/runner.py +++ b/src/openultrasast/plane/runner.py @@ -25,8 +25,7 @@ The command runs as a child process, ``python -m openultrasast.plane.tasks./.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM, as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat @@ -48,13 +47,9 @@ Environment the runner reads, all optional except the two ax variables: OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10 OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout) OUSAST_GIT_PINS JSON ``{"/": "<40-hex commit>"}`` from the reconciler - OUSAST_ARTIFACT_URL, OUSAST_RUN where to deliver, and the run name carried in ``X-Ousast-Run`` - OUSAST_ARTIFACT_DIAL ``host:port`` to connect to instead of the URL's host, which is sent as - ``Host`` (the egress gateway decides on and resolves that name) OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry OUSAST_INPUTS JSON ``{"": "/"}``: after the start and before the - command each is fetched from ``delivery.inputs[NAME]`` (else from - ``/inputs/`` with the receiver's dial and headers) + command each is fetched from ``delivery.inputs[NAME]`` to ``OUSAST_INPUT_``, with the same retries as delivery; a failed fetch delivers a failed summary naming the input, the command never runs @@ -700,16 +695,6 @@ def build_tar(output_dir: Path) -> bytes: return buffer.getvalue() -def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[str, dict[str, str]]: - """(URL to connect to, headers): with ``dial`` the connection goes there and the URL's host is sent as ``Host``.""" - out = dict(headers) - if not dial: - return url, out - parts = urlsplit(url) - out["Host"] = parts.netloc - return parts._replace(netloc=dial).geturl(), out - - def safe_url(url: str) -> str: """``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential).""" parts = urlsplit(url) @@ -723,18 +708,14 @@ def deliver( *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0, - dial: str | None = None, - method: str = "POST", ) -> int: - """Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry - with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query. + """PUT the payload with retries and doubling backoff; return the 2xx status. - ``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it - cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself. + Log lines and errors carry the URL without its query. """ - last, (target, headers) = "no attempt made", _dialled(url, headers, dial) + last = "no attempt made" for attempt in range(1, attempts + 1): - request = urllib.request.Request(target, data=payload, method=method, headers=headers) + request = urllib.request.Request(url, data=payload, method="PUT", headers=headers) try: with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL status = int(response.status) @@ -752,16 +733,16 @@ def deliver( def fetch( - url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0, dial: str | None = None + url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0 ) -> int: """GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`. Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed.""" - last, (target, headers) = "no attempt made", _dialled(url, headers, dial) + last = "no attempt made" dest.parent.mkdir(parents=True, exist_ok=True) partial = dest.with_name(dest.name + ".part") for attempt in range(1, attempts + 1): - request = urllib.request.Request(target, method="GET", headers=headers) + request = urllib.request.Request(url, method="GET", headers=headers) try: with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310 shutil.copyfileobj(response, out, 1 << 16) @@ -784,7 +765,7 @@ def fetch( def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]: """Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_`` path: from ``delivery.inputs[NAME]`` (a - presigned GET, no headers) when the start request carries one, else from the receiver. + presigned GET, no headers) carried by the start request. :class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing.""" raw = environ.get("OUSAST_INPUTS") @@ -796,21 +777,15 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None if not isinstance(inputs, dict): raise RunnerError("OUSAST_INPUTS must map input names to /") - url = environ.get("OUSAST_ARTIFACT_URL") - headers = {"X-Ousast-Run": environ.get("OUSAST_RUN", ""), "X-Ousast-Task": environ.get("OUSAST_TASK", "")} backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0")) fetched: list[Path] = [] for name, ref in sorted(inputs.items()): dest = environ.get(f"OUSAST_INPUT_{name}") presigned = delivery.inputs.get(name) if delivery is not None else None - if not dest or not isinstance(ref, str) or not (presigned or url): - raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}") + if not dest or not isinstance(ref, str) or not presigned: + raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL or OUSAST_INPUT_{name}") try: - if presigned: - size = fetch(presigned, {}, Path(dest), backoff=backoff) - else: - source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}" - size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None) + size = fetch(presigned, {}, Path(dest), backoff=backoff) except DeliveryError as exc: raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None log.info("input %s (%s): %d bytes to %s", name, ref, size, dest) @@ -818,23 +793,14 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = return fetched -def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool: - """PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver; +def _deliver_output(output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool: + """PUT the output tar to ``delivery.put`` when the start request carried one; False when there is nowhere to deliver; :class:`DeliveryError` when it fails.""" backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0")) if delivery is not None and delivery.put: - deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT") + deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff) return True - url = environ.get("OUSAST_ARTIFACT_URL") - if not url: - return False - # the Run's task name (``OUSAST_TASK``) is what the receiver knows; the ax Task is named ``-`` - headers = {"Content-Type": "application/x-tar", "X-Ousast-Task": environ.get("OUSAST_TASK") or task.metadata.name} - run = environ.get("OUSAST_RUN") - if run: - headers["X-Ousast-Run"] = run - deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None) - return True + return False # --- entrypoint -------------------------------------------------------------------------------------------------- @@ -849,7 +815,7 @@ class _Session: @property def marker(self) -> Path: - """``/.done.json``: the command finished; ``delivered`` says whether the receiver has it.""" + """``/.done.json``: the command finished; ``delivered`` says whether the store has it.""" return self.state_dir / f"{self.task.metadata.name}.done.json" @property @@ -893,7 +859,7 @@ def _finish( """ marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)} try: - marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery) + marker["delivered"] = _deliver_output(session.output_dir, env, delivery) except DeliveryError as exc: log.error("artifact delivery failed: %s", exc) marker.update(status="failed", reason=f"artifact delivery failed: {exc}") @@ -938,7 +904,7 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa return 0 try: _materialise_once(session, env) - fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only) + fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it except RunnerError as exc: log.error("runner failed: %s", exc) failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir)) diff --git a/src/openultrasast/plane/tasks/features.py b/src/openultrasast/plane/tasks/features.py index ddf36b2..0c5c5be 100644 --- a/src/openultrasast/plane/tasks/features.py +++ b/src/openultrasast/plane/tasks/features.py @@ -199,22 +199,6 @@ def _model(summary: object) -> str | None: return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None -def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]: - """:func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present).""" - final = base / f"{case}-final" / "agreed.json" - return case_features( - facts=_json(base / f"{case}-facts" / "facts.json"), - passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES}, - models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES}, - agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"), - alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"), - alerts_summary=_json(base / f"{case}-alerts" / "summary.json"), - workspace=workspace, - facts_summary=_json(base / f"{case}-facts" / "summary.json"), - model_roles=_json(base / f"{case}-roles" / "roles.json"), - ) - - def _path(env: Mapping[str, str], name: str) -> Path | None: return Path(env[name]) if env.get(name) else None @@ -266,7 +250,7 @@ def main(environ: Mapping[str, str] | None = None) -> int: return 0 if summary["status"] == "done" else 2 -__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"] +__all__ = ["DROPPED_ON_READ", "case_features", "main", ] if __name__ == "__main__": diff --git a/src/openultrasast/rank.py b/src/openultrasast/rank.py index 9452d1a..bbf6629 100644 --- a/src/openultrasast/rank.py +++ b/src/openultrasast/rank.py @@ -4,7 +4,7 @@ import json from collections.abc import Collection from dataclasses import asdict, dataclass, replace from pathlib import Path -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING from .preprocess import FileTarget @@ -28,11 +28,6 @@ class RankingScore: static_boosts: list[str] -class RankerClient(Protocol): - def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object: - raise NotImplementedError - - def rank_targets(targets: list[FileTarget], model_rankings: object | None = None, model_id: str | None = None) -> list[RankingScore]: model_by_path = parse_ranker_response(model_rankings, model_id) if model_rankings is not None else {} rankings: list[RankingScore] = [] @@ -43,32 +38,6 @@ def rank_targets(targets: list[FileTarget], model_rankings: object | None = None return sorted(rankings, key=lambda item: (-item.priority, item.path)) -def rank_targets_with_model( - targets: list[FileTarget], - *, - client: RankerClient, - model: str, - chunk_size: int = 150, - timeout_seconds: int = 60, -) -> list[RankingScore]: - if chunk_size < 1: - raise RankingError("chunk_size must be at least 1") - model_rankings: list[dict[str, object]] = [] - for start in range(0, len(targets), chunk_size): - chunk = targets[start : start + chunk_size] - response = client.complete_json( - model=model, - messages=[ - {"role": "system", "content": _ranker_system_prompt()}, - {"role": "user", "content": json.dumps([_target_prompt_payload(target) for target in chunk], sort_keys=True)}, - ], - timeout_seconds=timeout_seconds, - ) - for ranking in parse_ranker_response(response, model).values(): - model_rankings.append(asdict(ranking)) - return rank_targets(targets, {"rankings": model_rankings}, model_id=model) - - def write_rankings(rankings: list[RankingScore], path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) payload = {"rankings": [asdict(ranking) for ranking in rankings]} @@ -205,25 +174,6 @@ def _clamp_score(value: int) -> int: return min(5, max(1, value)) -def _ranker_system_prompt() -> str: - return ( - "Score each file for security review priority. Return JSON only with a rankings array. " - "Each item must include path, surface, influence, reachability, rationale, and optional static_boosts. " - "Scores must be integers from 1 to 5." - ) - - -def _target_prompt_payload(target: FileTarget) -> dict[str, object]: - return { - "path": target.path, - "language": target.language, - "loc": target.loc, - "tags": target.tags, - "has_fuzz_entry_point": target.has_fuzz_entry_point, - "reachability_hints": target.reachability_hints, - } - - def _reachability_from_hints(hints: list[dict[str, object]]) -> int: score = 0 for hint in hints: diff --git a/src/openultrasast/search/coordinator.py b/src/openultrasast/search/coordinator.py index c3a0c84..855995c 100644 --- a/src/openultrasast/search/coordinator.py +++ b/src/openultrasast/search/coordinator.py @@ -7,20 +7,14 @@ call is conservatively charged its reservation on resume, never silently free. from __future__ import annotations -import argparse -import hashlib -import json import re import time from collections import Counter from collections.abc import Callable from dataclasses import asdict, dataclass, replace -from importlib import import_module -from pathlib import Path from typing import Any, Protocol from ..plane.budget import BudgetExhausted, SpendBudget -from ..plane.memory import open_store from .board import Board from .budget import SearchBudget @@ -421,79 +415,3 @@ class Coordinator: @staticmethod def _refs(refs: Any) -> bool: return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs) - - -class BatchExecutor: - """Adapter to the existing AX/VM transfer and checkpoint protocol. - - Pass AXLane/DockerLane instances and a trusted worker command. No default - worker is invented here: reason/explore entrypoints belong to group 5. - The coordinator owns retries, so batch performs exactly one attempt. - """ - - def __init__(self, *, image: str, command: tuple[str, ...], output: Path, lanes: dict[str, Any]) -> None: - batch = import_module("benchmarks.ax.batch") - batch.validate_image(image) - if not command: - raise ValueError("worker command required") - self.image, self.command, self.output, self.lanes = image, command, output, lanes - - def __call__(self, task: SearchTask) -> dict[str, Any]: - import copy - - batch = import_module("benchmarks.ax.batch") - - if task.lane not in self.lanes: - return {"status": "sandbox_failure", "cost_usd": 0} - data = json.dumps(asdict(task), sort_keys=True, allow_nan=False).encode() - digest = hashlib.sha256(data).hexdigest() - identity = task.search_id + "-" + task.id - item = batch.Item( - identity, - {"SPEC_URL": data}, - "result.dat", - self.command, - {"SEARCH_STEP": task.step, "SEARCH_ID": task.search_id, "SEARCH_TASK_ID": task.id}, - digest, - ) - - def validate(raw: bytes, output: Path) -> None: - if len(raw) > 262144: - raise ValueError("search result size exceeded") - record = json.loads(raw) - if not isinstance(record, dict) or record.get("status") not in ( - "ok", - "sandbox_failure", - "instrument_failure", - "oom", - "execution_failure", - ): - raise ValueError("invalid search result") - (output / "result.json").write_text(json.dumps(record)) - - workload = batch.Workload(self.image, frozenset({"SPEC_URL", "RESULT_URL"}), task.limits["task_wall_seconds"], validate) - lane = copy.copy(self.lanes[task.lane]) - if hasattr(lane, "workload"): - lane.workload = workload - out = self.output / task.search_id / task.id - batch.dispatch([item], workload, out, {task.lane: lane}, lanes=(task.lane,), max_attempts=1) - record_path = out / (hashlib.sha256(identity.encode()).hexdigest() + ".json") - if not record_path.exists(): - return {"status": "execution_failure", "cost_usd": 0} - record: dict[str, Any] = json.loads(record_path.read_text()) - return record - - -def main() -> None: - """Offline checkpoint inspection, also useful before resuming with an executor.""" - parser = argparse.ArgumentParser(description="Inspect a search board checkpoint (no model calls)") - parser.add_argument("--memory", required=True) - parser.add_argument("--search-id", required=True) - parser.add_argument("--head", required=True) - args = parser.parse_args() - board = Board.resume(open_store(args.memory), args.search_id, args.head, coordinator="inspect") - print(board.snapshot(), end="") - - -if __name__ == "__main__": - main() diff --git a/src/openultrasast/search/verify.py b/src/openultrasast/search/verify.py index 041a88e..3ce2fe6 100644 --- a/src/openultrasast/search/verify.py +++ b/src/openultrasast/search/verify.py @@ -22,7 +22,7 @@ import zipfile from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from contextlib import suppress -from dataclasses import asdict, dataclass, field, replace +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, cast @@ -709,20 +709,3 @@ def verify( isolation_mode=mode, chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)), ) - - -def main() -> None: - import argparse - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("affected", type=Path) - parser.add_argument("safe", type=Path) - parser.add_argument("artefact", type=Path) - parser.add_argument("family") - args = parser.parse_args() - result = verify(Side(args.affected), Side(args.safe), args.artefact, args.family) - print(json.dumps(asdict(result), indent=2)) - - -if __name__ == "__main__": - main() diff --git a/src/openultrasast/search/worker.py b/src/openultrasast/search/worker.py index 682742a..4fb8066 100644 --- a/src/openultrasast/search/worker.py +++ b/src/openultrasast/search/worker.py @@ -2,11 +2,9 @@ from __future__ import annotations -import argparse import copy import json import math -import os import time from pathlib import Path from typing import Any, Protocol, cast @@ -15,12 +13,11 @@ import yaml from ..model.endpoint import Prices from ..plane.budget import BudgetExhausted, SpendBudget, cost_of -from ..provider.openrouter import OpenRouterChatClient, parse_json_content +from ..provider.openrouter import parse_json_content from . import _sandbox from .budget import SearchBudget from .coordinator import SearchTask from .demo import DEMO_SCHEMA, FAMILY_ORACLES, MAX_DEMO_BYTES, load_demo, validate_demo, validate_oracle -from .executor import ObjectStoreExecutor from .task_storage import exception_reason @@ -525,44 +522,3 @@ class Worker: "cost_usd": meter.spent, "isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"), } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--task", required=True, type=Path) - parser.add_argument("--repo", required=True, type=Path) - parser.add_argument("--demo", required=True, type=Path) - parser.add_argument("--model", required=True) - parser.add_argument("--input-per-m", required=True, type=float) - parser.add_argument("--output-per-m", required=True, type=float) - parser.add_argument("--command-put-url") - parser.add_argument("--result-get-url") - args = parser.parse_args() - if bool(args.command_put_url) != bool(args.result_get_url): - parser.error("both executor mailbox URLs are required") - executor = ObjectStoreExecutor(args.command_put_url, args.result_get_url) if args.command_put_url else None - task = SearchTask(**json.loads(args.task.read_text())) - if task.step == "explore" and executor is None: - parser.error("explore requires a separate repository executor") - client = OpenRouterChatClient( - api_key=os.environ["OPENROUTER_API_KEY"], - base_url=os.environ.get("OPENROUTER_BASE_URL", OpenRouterChatClient.base_url), - max_attempts=1, - ) - worker = Worker( - client, - repo=args.repo, - demo=args.demo, - model=args.model, - prices=Prices(args.input_per_m, args.input_per_m, args.output_per_m), - executor=executor, - ) - try: - print(json.dumps(worker(task))) - finally: - if executor is not None: - executor.submit("stop", {}, timeout_seconds=10) - - -if __name__ == "__main__": - main() diff --git a/src/openultrasast/tool_hunter.py b/src/openultrasast/tool_hunter.py index 9e77724..2e5bd0b 100644 --- a/src/openultrasast/tool_hunter.py +++ b/src/openultrasast/tool_hunter.py @@ -15,12 +15,8 @@ from .findings import StaticFinding from .hunter_tools import ( PathEscapesRepo, clamp_repo_path, - entry_points, find_refs, - flows, grep_repo, - obligations, - read_definition, read_file, ) from .provider.openrouter import OpenRouterChatClient, OpenRouterError @@ -31,7 +27,6 @@ DEFAULT_MAX_STEPS = 4 CLIENT_ENV = "OPENULTRASAST_HUNTER_CLIENT" _FINDING_ID_PREFIX = "tool-hunter:" DEFAULT_TOOL_NAMES = ("read_file", "grep_repo", "find_refs") -_HUNTER_TOOL_NAMES = frozenset(DEFAULT_TOOL_NAMES) | {"read_definition", "entry_points", "flows", "obligations"} _SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"}) _SCRIPTED_FLAGS = frozenset({"scripted", "script"}) _DUMP_FLAGS = frozenset({"dump", "dump-only"}) @@ -91,50 +86,6 @@ HUNTER_TOOLS: list[dict[str, object]] = [ }, }, }, - { - "type": "function", - "function": { - "name": "read_definition", - "description": "Resolve one named function to its file, line span and body. Use it instead of guessing what a callee does.", - "parameters": { - "type": "object", - "properties": {"symbol": {"type": "string"}, "path": {"type": "string"}, "max_chars": {"type": "integer"}}, - "required": ["symbol"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "entry_points", - "description": "List the handlers this repository exposes, with their access classification. Identifiers and line spans only.", - "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "flows", - "description": "List adjudicated source-to-sink records inside one function. Identifiers and line numbers only.", - "parameters": { - "type": "object", - "properties": {"path": {"type": "string"}, "function": {"type": "string"}}, - "required": ["path", "function"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "obligations", - "description": "List what one function owes and has not discharged: the operation, the missing discharger and why it is owed.", - "parameters": { - "type": "object", - "properties": {"path": {"type": "string"}, "function": {"type": "string"}}, - "required": ["path", "function"], - }, - }, - }, ] _SYSTEM_PROMPT = ( @@ -330,15 +281,13 @@ def run_tool_hunter( system_prompt: str = _SYSTEM_PROMPT, user_prompt: str | None = None, context_files: Sequence[str] = (), - tools: Sequence[str] | None = None, max_chars: int = _DEFAULT_MAX_CHARS, tags: Sequence[str] = ("tool-hunter",), ) -> list[StaticFinding]: - """One bounded hunt. Every keyword defaults to today's behaviour; a per-family detector overrides the - prompt, the context it starts from, the tools it may call and the tags its findings carry.""" + """One bounded hunt with a prompt, starting context and finding tags.""" if max_steps <= 0: return [] - schemas = _tool_schemas(tools) + schemas = HUNTER_TOOLS messages: list[dict[str, object]] = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": _user_message(root, hotspots, user_prompt, context_files, max_chars)}, @@ -350,7 +299,7 @@ def run_tool_hunter( if tool_calls: messages.append(_assistant_tool_message(response.content, tool_calls, response.reasoning)) for call in tool_calls: - if call.name in _HUNTER_TOOL_NAMES: + if call.name in DEFAULT_TOOL_NAMES: used_tools = True messages.append(_tool_result_message(root, call, max_chars)) continue @@ -414,12 +363,6 @@ def _looks_like_json(content: str) -> bool: return stripped.startswith(("[", "{")) -def _tool_schemas(names: Sequence[str] | None) -> list[dict[str, object]]: - wanted = tuple(names) if names is not None else DEFAULT_TOOL_NAMES - by_name = {str(schema["function"]["name"]): schema for schema in HUNTER_TOOLS} # type: ignore[index] - return [by_name[name] for name in wanted if name in by_name] - - def _user_message(root: Path, hotspots: Sequence[Hotspot], user_prompt: str | None, context_files: Sequence[str], max_chars: int) -> str: # Context files FIRST, the question LAST. DeepSeek bills a cached prefix at 1/30 of a fresh one, and the cache # is by prefix: with the question first, every candidate in the same file broke the prefix at its first @@ -502,30 +445,13 @@ def _dispatch_tool(root: Path, name: str, arguments: Mapping[str, object], defau path = _require_str(arguments, "path") max_chars = _optional_int(arguments, "max_chars", default_max_chars) return {"path": path, "content": read_file(root, path, max_chars=max_chars)} - if name == "read_definition": - symbol = _require_str(arguments, "symbol") - scope = arguments.get("path") - found = read_definition( - root, - symbol, - max_chars=_optional_int(arguments, "max_chars", default_max_chars), - path=str(scope) if isinstance(scope, str) and scope else None, - ) - return found if found is not None else {"symbol": symbol, "found": False} - if name == "entry_points": - scope = arguments.get("path") - return entry_points(root, path=str(scope) if isinstance(scope, str) and scope else None) - if name == "flows": - return flows(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function")) - if name == "obligations": - return obligations(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function")) if name == "grep_repo": pattern = _require_str(arguments, "pattern") max_matches = _optional_int(arguments, "max_matches", _DEFAULT_MAX_MATCHES) return grep_repo(root, pattern, max_matches=max_matches) if name == "find_refs": symbol = _require_str(arguments, "symbol") - return find_refs(root, symbol, None) + return find_refs(root, symbol) raise ValueError(f"unknown tool: {name}") diff --git a/src/openultrasast/verification.py b/src/openultrasast/verification.py index 170ebe6..14c25ee 100644 --- a/src/openultrasast/verification.py +++ b/src/openultrasast/verification.py @@ -36,16 +36,6 @@ class VerificationResult: context_sources: list[str] -def validate_evidence_transition(current: EvidenceLevel | str, target: EvidenceLevel | str) -> EvidenceLevel: - current_level = EvidenceLevel(current) - target_level = EvidenceLevel(target) - if _EVIDENCE_ORDER[target_level] < _EVIDENCE_ORDER[current_level]: - raise ValueError(f"cannot move evidence level backward: {current_level} -> {target_level}") - if _EVIDENCE_ORDER[target_level] - _EVIDENCE_ORDER[current_level] > 1: - raise ValueError(f"cannot skip evidence levels: {current_level} -> {target_level}") - return target_level - - def is_report_verified(evidence_level: EvidenceLevel | str, status: VerificationStatus | str) -> bool: level = EvidenceLevel(evidence_level) verification_status = VerificationStatus(status) diff --git a/tests/test_calibration.py b/tests/test_calibration.py index 6929f68..e7d5e96 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -6,7 +6,6 @@ from openultrasast.calibration import ( apply_ranking_calibration, calibrate_ranking, calibrate_rankings, - compute_ranking_metrics, learning_from_finding_outcome, record_false_positive_learning, write_false_positive_learnings, @@ -62,34 +61,6 @@ def test_false_positive_learning_records_reason_and_scope(tmp_path: Path) -> Non assert "missing_attacker_control" in output.read_text() -def test_ranking_metrics_track_false_positive_reasons_by_tier() -> None: - metrics = compute_ranking_metrics( - [ - {"tier": "A", "outcome": FindingOutcome.VERIFIED, "time_to_verification_seconds": 10}, - { - "tier": "A", - "outcome": FindingOutcome.REJECTED, - "reason": FalsePositiveReason.UNREACHABLE_PATH, - "time_to_verification_seconds": 20, - }, - {"tier": "B", "outcome": FindingOutcome.REJECTED, "reason": FalsePositiveReason.DUPLICATE}, - { - "tier": "A", - "outcome": FindingOutcome.UNVERIFIED, - "reason": FalsePositiveReason.UNVERIFIED, - "missed_fixture_vulnerabilities": 1, - }, - ], - tier="A", - ) - - assert metrics.verified_findings == 1 - assert metrics.false_positives == 2 - assert metrics.time_to_verification_seconds == 15.0 - assert metrics.missed_fixture_vulnerabilities == 1 - assert metrics.false_positive_reasons["unreachable_path"] == 1 - - def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None: learning = record_false_positive_learning( _finding("auth/login.py"), diff --git a/tests/test_findings.py b/tests/test_findings.py index 49f0902..676a572 100644 --- a/tests/test_findings.py +++ b/tests/test_findings.py @@ -1,7 +1,7 @@ import json from pathlib import Path -from openultrasast.findings import build_quick_hunter_prompt, quick_scan_findings, write_findings +from openultrasast.findings import quick_scan_findings, write_findings from openultrasast.mapping import analyze_entry_points, attach_reachability_hints from openultrasast.preprocess import preprocess_repository from openultrasast.rank import rank_targets @@ -92,17 +92,3 @@ def test_findings_inherit_only_matching_function_level_reachability(tmp_path: Pa assert unreachable.ranking_priority <= 1.5 -def test_quick_hunter_prompt_is_bounded() -> None: - repo = Path("/repo/example.py") - target = next(iter(preprocess_repository(repo.parent)[1]), None) if repo.parent.exists() else None - assert target is None - - class Target: - path = "example.py" - language = "python" - tags = ["auth_boundary"] - - prompt = build_quick_hunter_prompt(Target(), "x" * 5000) # type: ignore[arg-type] - - assert "example.py" in prompt - assert len(prompt) < 4300 diff --git a/tests/test_fusion.py b/tests/test_fusion.py index 3cbf66b..b29757c 100644 --- a/tests/test_fusion.py +++ b/tests/test_fusion.py @@ -6,13 +6,7 @@ from pathlib import Path import pytest from openultrasast.findings import StaticFinding -from openultrasast.fusion import ( - FusionDisposition, - fuse_finding, - fuse_findings, - fusion_triggers, - should_fuse, -) +from openultrasast.fusion import FusionDisposition, fuse_finding, fuse_findings, fusion_triggers from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus @@ -63,10 +57,6 @@ def test_triggers_on_high_severity_and_reachable() -> None: assert "gates_risky_fix" in triggers -def test_low_severity_unreachable_does_not_trigger() -> None: - assert not should_fuse(_finding(severity="low", reachability="unknown")) - - def test_verifier_disagreement_triggers() -> None: triggers = fusion_triggers( _finding(severity="low", reachability="unknown"), @@ -83,10 +73,6 @@ def test_evidence_conflict_triggers_when_rejected_but_reachable() -> None: assert "evidence_conflict" in triggers -def test_high_assurance_request_forces_a_trigger() -> None: - assert should_fuse(_finding(severity="low", reachability="unknown"), high_assurance=True) - - # ---- deterministic two-panel decider + dispositions ------------------------- diff --git a/tests/test_harness.py b/tests/test_harness.py index 89eebbd..06eb880 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1,100 +1,36 @@ import json -from dataclasses import dataclass from pathlib import Path -from typing import Any - -import pytest from openultrasast.config import load_config -from openultrasast.harness import ( - HarnessContractError, - HarnessEvent, - HarnessRuntime, - HarnessTraceWriter, - ProcessorSpec, - write_harness_config, -) +from openultrasast.harness import HarnessRuntime, HarnessTraceWriter, write_harness_config -@dataclass -class RecordingProcessor: - spec: ProcessorSpec - - def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]: - return {"seen_stage": event.stage} - - -@dataclass -class BadWriteProcessor: - spec: ProcessorSpec - - def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]: - return {"undeclared": event.event_type} - - -def test_runtime_emits_stage_events_and_processor_state(tmp_path: Path) -> None: +def test_runtime_emits_stage_events(tmp_path: Path) -> None: trace_path = tmp_path / "events.jsonl" - processor = RecordingProcessor(ProcessorSpec(name="recorder", version="1", handles=("stage_start",), writes=("seen_stage",))) runtime = HarnessRuntime( scan_id="scan-1", config=load_config(None), trace_writer=HarnessTraceWriter(trace_path), - processors=[processor], ) result = runtime.run_stage("preprocess", lambda: "ok") events = [json.loads(line) for line in trace_path.read_text().splitlines()] assert result == "ok" - assert runtime.state["seen_stage"] == "preprocess" - assert [event["event_type"] for event in events] == ["stage_start", "stage_start", "stage_end"] - assert events[1]["payload"] == {"changed_state": ["seen_stage"], "processor": "recorder"} + assert [event["event_type"] for event in events] == ["stage_start", "stage_end"] -def test_strict_contract_mode_rejects_undeclared_writes(tmp_path: Path) -> None: - processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=())) - runtime = HarnessRuntime( - scan_id="scan-1", - config=load_config(None), - trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"), - processors=[processor], - ) - - with pytest.raises(HarnessContractError, match="undeclared writes"): - runtime.emit("stage_start", "rank", {}) - - -def test_warn_contract_mode_records_degradation(tmp_path: Path) -> None: - processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=())) - runtime = HarnessRuntime( - scan_id="scan-1", - config=load_config(None), - trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"), - processors=[processor], - contract_mode="warn", - ) - - runtime.emit("stage_start", "rank", {}) - - assert "undeclared writes" in runtime.state["degradations"][0] - assert '"status": "degraded"' in (tmp_path / "events.jsonl").read_text() - - -def test_harness_config_serializes_models_processors_and_prompt_hashes(tmp_path: Path) -> None: +def test_harness_config_serializes_models_and_prompt_hashes(tmp_path: Path) -> None: config_path = tmp_path / "openultrasast.toml" config_path.write_text('[models]\nranker = "openrouter/ranker"\n') - processor = RecordingProcessor(ProcessorSpec(name="recorder", version="2026-01-01", handles=("stage_start",), writes=("seen_stage",))) output = tmp_path / "harness.json" write_harness_config( config=load_config(config_path), - processors=[processor], - contract_mode="strict", path=output, prompts={"ranker": "score files"}, ) payload = json.loads(output.read_text()) assert payload["model_roles"]["ranker"] == "openrouter/ranker" - assert payload["processor_versions"] == {"recorder": "2026-01-01"} assert len(payload["prompt_hashes"]["ranker"]) == 64 diff --git a/tests/test_hunter_curated_tools.py b/tests/test_hunter_curated_tools.py deleted file mode 100644 index 002a9a5..0000000 --- a/tests/test_hunter_curated_tools.py +++ /dev/null @@ -1,198 +0,0 @@ -"""learning-harness task 1.5: curated repository tools and hunter-loop parameters (offline).""" - -from __future__ import annotations - -import json -import tempfile -from pathlib import Path - -import pytest - -from openultrasast.hunter_tools import PathEscapesRepo -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n" - "def helper(value):\n" - " return value.strip()\n" -) - - -@pytest.fixture -def repo(tmp_path: Path) -> Path: - (tmp_path / "app.py").write_text(APP) - return tmp_path - - -def test_read_definition_returns_one_span_and_refuses_to_leave_the_repository(repo: Path) -> None: - from openultrasast.hunter_tools import read_definition - - found = read_definition(repo, "leaky", max_chars=4000) - assert found is not None - assert found["path"] == "app.py" and found["name"] == "leaky" - assert found["start_line"] == 10 and found["end_line"] == 13 - assert str(found["text"]).startswith("@app.route('/books/any/<title>')") # a handler's decorators come with it - assert "def leaky" in str(found["text"]) and "def guarded" not in str(found["text"]) - assert read_definition(repo, "not_a_function", max_chars=4000) is None - with pytest.raises(PathEscapesRepo): - read_definition(repo, "leaky", max_chars=4000, path="../outside.py") - - -def test_entry_points_lists_handlers_with_their_access_and_never_source_text(repo: Path) -> None: - from openultrasast.hunter_tools import entry_points - - rows = entry_points(repo) - by_function = {str(row["function"]): row for row in rows if row.get("function")} - assert {"guarded", "leaky"} <= set(by_function) - assert by_function["guarded"]["access_level"] == "authenticated" and by_function["leaky"]["access_level"] == "public" - assert by_function["leaky"]["path"] == "app.py" and isinstance(by_function["leaky"]["line"], int) - assert all("text" not in row and "content" not in row for row in rows) # identifiers and spans only - assert entry_points(repo, path="app.py") == rows - with pytest.raises(PathEscapesRepo): - entry_points(repo, path="../elsewhere.py") - - -def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path) -> None: - from openultrasast.hunter_tools import obligations - - rows = obligations(repo, path="app.py", function="leaky") - assert rows and all(row["function"] == "leaky" for row in rows) - assert {"protected_read"} == {str(row["operation"]) for row in rows} - assert "identity_constraint" in {str(row["missing"]) for row in rows} - assert all("text" not in row and "content" not in row for row in rows) - assert obligations(repo, path="app.py", function="helper") == [] - with pytest.raises(PathEscapesRepo): - obligations(repo, path="../outside.py", function="leaky") - - -def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None: - from openultrasast.hunter_tools import flows - - (repo / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n") - rows = flows(repo, path="flow.py", function="run") - assert all(row["function"] == "run" and row["path"] == "flow.py" for row in rows) - assert all(set(row) <= {"path", "function", "line", "sources", "sinks", "disposition", "cwe"} for row in rows) - assert flows(repo, path="app.py", function="helper") == [] - with pytest.raises(PathEscapesRepo): - flows(repo, path="/etc/passwd", function="run") - - -def test_the_loop_keeps_todays_behaviour_by_default(repo: Path) -> None: - from openultrasast.complexity.map import Hotspot - from openultrasast.tool_hunter import DEFAULT_TOOL_NAMES - - hotspot = Hotspot( - path="app.py", - function_name=None, - score=1.0, - band="high", - signals={}, - rationale="fixture", - test_hint=None, - inventory_finding_ids=(), - ) - client = ScriptedChatClient( - [ - ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)), - ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'), - ] - ) - findings = run_tool_hunter(repo, [hotspot], client=client, model="m", max_steps=4) - assert [finding.finding_id for finding in findings] == ["tool-hunter:app.py:12"] - assert findings[0].tags == ["tool-hunter"] and findings[0].evidence_level == "suspicion" - assert tuple(tool["function"]["name"] for tool in client.calls[0]["tools"]) == DEFAULT_TOOL_NAMES - - -def test_the_loop_takes_a_prompt_context_tools_and_tags_and_replays_reasoning(repo: Path) -> None: - from openultrasast.complexity.map import Hotspot - - hotspot = Hotspot( - path="app.py", - function_name="leaky", - score=1.0, - band="high", - signals={}, - rationale="fixture", - test_hint=None, - inventory_finding_ids=(), - ) - client = ScriptedChatClient( - [ - ChatResponse( - content="looking", - reasoning="first I check the route", - tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),), - ), - ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'), - ] - ) - findings = run_tool_hunter( - repo, - [hotspot], - client=client, - model="m", - max_steps=4, - system_prompt="You hunt access control only.", - user_prompt="Look at leaky.", - context_files=("app.py",), - tools=("read_definition", "obligations"), - tags=("family:access_control", "detector:access_control@1"), - ) - first = client.calls[0] - assert first["messages"][0]["content"] == "You hunt access control only." - assert "Look at leaky." in str(first["messages"][1]["content"]) and "def leaky" in str(first["messages"][1]["content"]) - assert [tool["function"]["name"] for tool in first["tools"]] == ["read_definition", "obligations"] - replayed = [message for message in client.calls[1]["messages"] if message.get("role") == "assistant"] - assert replayed and replayed[0]["reasoning_content"] == "first I check the route" - tool_reply = json.loads(str(client.calls[1]["messages"][-1]["content"])) - assert tool_reply and tool_reply[0]["missing"] == "identity_constraint" - assert findings[0].tags == ["family:access_control", "detector:access_control@1"] - - -def test_the_loop_asks_once_more_for_the_answer_instead_of_dropping_prose() -> None: - """Measured live: the model spends its step budget on tools and then answers in prose, and the whole hunt is - discarded (learning-harness Req 6.3). - - Four steps, three of them tool calls, and the fourth a paragraph that names the sink, the argument and the - line — parsed as nothing. A detector that found the bug and reported zero is indistinguishable from one that - found nothing, and every family in round zero would have baselined at zero for this reason alone.""" - from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter - - prose = "The function passes `command` straight to os.system at line 2, an OS command sink." - answer = json.dumps([{"path": "app.py", "line": 2, "title": "command injection", "rationale": "os.system(command)"}]) - client = ScriptedChatClient( - [ - ChatResponse(tool_calls=(ToolCall(id="c1", name="grep_repo", arguments={"pattern": "system"}),)), - ChatResponse(tool_calls=(ToolCall(id="c2", name="read_file", arguments={"path": "app.py"}),)), - ChatResponse(content=prose), - ChatResponse(content=answer), # the one extra turn, asked for explicitly - ] - ) - with tempfile.TemporaryDirectory() as scratch: - root = Path(scratch) - (root / "app.py").write_text("import os\ndef run(command):\n return os.system(command)\n") - findings = run_tool_hunter(root, [], client=client, model="m", max_steps=3) - assert [item.line for item in findings] == [2] - final = client.calls[-1] - # Asked for JSON in the prompt, not through the provider's JSON mode: with tools attached, DeepSeek's JSON mode - # returned `{"type": "json_object"}` and lost the answers (tests/test_tool_hunter.py pins the shapes). - assert not final.get("json_object") and final.get("tools") == [], "the last turn is plain text without tools" - text = "\n".join(str(message.get("content") or "") for message in final["messages"]) - assert "JSON" in text and prose in text, "the prose it already wrote is what the final turn is asked to format" - - -def test_a_hunt_that_never_called_a_tool_still_reports_nothing() -> None: - """The extra turn is for formatting an answer, never for inventing one.""" - from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, run_tool_hunter - - client = ScriptedChatClient([ChatResponse(content="I think there is an injection here."), ChatResponse(content="[]")]) - with tempfile.TemporaryDirectory() as scratch: - root = Path(scratch) - (root / "app.py").write_text("x = 1\n") - assert run_tool_hunter(root, [], client=client, model="m", max_steps=3) == [] - assert len(client.calls) == 1, "no tool was called, so there is nothing to format" diff --git a/tests/test_hunter_tools.py b/tests/test_hunter_tools.py index 0d075c9..f49d833 100644 --- a/tests/test_hunter_tools.py +++ b/tests/test_hunter_tools.py @@ -3,7 +3,6 @@ from pathlib import Path import pytest from openultrasast.hunter_tools import PathEscapesRepo, clamp_repo_path, find_refs, grep_repo, read_file -from openultrasast.mapping import EntryPointRecord SPLIT_SINK = """\ term = request.args["q"] @@ -85,42 +84,13 @@ def test_read_file_truncates_and_rejects_symlink_escape(tmp_path: Path) -> None: read_file(root, "link.py", max_chars=100) -def test_find_refs_uses_mapping_index_and_clamped_text_search(tmp_path: Path) -> None: +def test_find_refs_searches_source_text(tmp_path: Path) -> None: root = _repo_with_split_sink(tmp_path) - mapping_index = [ - {"path": "app.py", "name": "query", "line": 2}, - EntryPointRecord( - path="app.py", - line=3, - end_line=3, - function_name="search", - name="search", - kind="route", - access_level="public", - trust_boundary="http_request", - access_evidence=[], - conditions=[], - provenance="test", - rationale="fixture", - ), - ] - - query_refs = find_refs(root, "query", mapping_index) - assert any(ref["path"] == "app.py" and ref["name"] == "query" for ref in query_refs) - assert any(ref.get("source") == "mapping" for ref in query_refs) - assert any(ref.get("source") == "text" and "query" in str(ref.get("text", ref.get("name", ""))) for ref in query_refs) - - search_refs = find_refs(root, "search", mapping_index) - assert any(ref["path"] == "app.py" and ref["name"] == "search" for ref in search_refs) - - -def test_find_refs_rejects_mapping_path_escape(tmp_path: Path) -> None: - root = _repo_with_split_sink(tmp_path) - - with pytest.raises(PathEscapesRepo): - find_refs(root, "passwd", [{"path": "../etc/passwd", "name": "passwd"}]) - with pytest.raises(PathEscapesRepo): - find_refs(root, "passwd", [{"path": "/etc/passwd", "name": "passwd"}]) + refs = find_refs(root, "query") + assert [ref["line"] for ref in refs] == [2, 3] + assert all(ref["path"] == "app.py" and ref["name"] == "query" and ref["source"] == "text" for ref in refs) + assert find_refs(root, "search") == [] + assert find_refs(root, "") == [] def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> None: diff --git a/tests/test_learn_folds.py b/tests/test_learn_folds.py index 6557d7a..d7be54a 100644 --- a/tests/test_learn_folds.py +++ b/tests/test_learn_folds.py @@ -5,17 +5,7 @@ from __future__ import annotations import pytest from learn_fixtures import corpus -from openultrasast.learn.folds import ( - INSUFFICIENT, - Fold, - FoldError, - assert_disjoint, - compile_split, - folds_digest, - leave_one_framework_out, - leave_one_source_out, - outer_folds, -) +from openultrasast.learn.folds import Fold, FoldError, assert_disjoint, compile_split, folds_digest, outer_folds def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None: @@ -26,8 +16,6 @@ def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4 folds = [ *outer_folds(examples, seed=1, exclude=split.groups), - *leave_one_source_out(examples, exclude=split.groups), - *leave_one_framework_out(examples, exclude=split.groups, min_groups=2), ] assert_disjoint(split, folds) for fold in folds: @@ -52,19 +40,6 @@ def test_folds_are_deterministic_by_seed() -> None: assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6)) -def test_leave_one_source_and_framework_out() -> None: - examples = corpus(groups=12) - sources = leave_one_source_out(examples) - assert {f.held_source for f in sources} == {"pairs", "population-v1", "population-v2"} - for fold in sources: - assert fold.eval_groups == {e.group for e in examples if e.source == fold.held_source} - frameworks = {f.held_framework: f for f in leave_one_framework_out(examples)} - assert set(frameworks) == {"flask", "django"} - assert all(f.status == INSUFFICIENT and not f.eval_groups for f in frameworks.values()) # 3 groups each < 10 - enough = {f.held_framework: f for f in leave_one_framework_out(examples, min_groups=3)} - assert enough["flask"].status == "ok" and enough["flask"].eval_groups == {e.group for e in examples if "flask" in e.frameworks} - - def test_a_shared_group_is_refused() -> None: examples = corpus(groups=8) split = compile_split(examples, seed=0) diff --git a/tests/test_learn_retrieve.py b/tests/test_learn_retrieve.py index 1e56897..8cc1f15 100644 --- a/tests/test_learn_retrieve.py +++ b/tests/test_learn_retrieve.py @@ -8,7 +8,7 @@ from dataclasses import replace import pytest from learn_fixtures import by_id, corpus, example, x_record -from openultrasast.learn.folds import Fold, compile_split, leave_one_framework_out, leave_one_source_out, outer_folds +from openultrasast.learn.folds import Fold, compile_split, outer_folds from openultrasast.learn.retrieve import ( BoundaryViolation, Target, @@ -166,13 +166,11 @@ def _all_folds(examples: list) -> list: split = compile_split(examples, seed=3) return [ *outer_folds(examples, seed=3, exclude=split.groups), - *leave_one_source_out(examples, exclude=split.groups), - *[f for f in leave_one_framework_out(examples, exclude=split.groups, min_groups=2) if f.status == "ok"], ] def test_boundary() -> None: - """For outer, leave-one-source-out and leave-one-framework-out folds, every retrieved example and every + """For outer folds, every retrieved example and every demonstration that reaches a prompt is eligible; ``seed_memory`` writes no example of the case's group.""" examples = corpus(groups=16) index = by_id(examples) @@ -186,7 +184,7 @@ def test_boundary() -> None: ) ] folds = _all_folds(examples) - assert {f.kind for f in folds} == {"outer", "leave_one_source_out", "leave_one_framework_out"} + assert {f.kind for f in folds} == {"outer"} checked = 0 for fold in folds: for target_example in [e for e in examples if e.group in fold.eval_groups]: @@ -206,19 +204,6 @@ def test_boundary() -> None: assert all(e.group != group for e in seed_memory(examples, group)) -def test_a_demonstration_of_the_held_out_framework_is_swapped_for_its_alternate() -> None: - flask = example("demo-flask", group="d/flask", label=1, frameworks=("flask",)) - alt = example("demo-plain", group="d/plain", label=1) - target_example = example("t", group="t/t", label=0, frameworks=("flask",)) - target = Target.of(target_example) - fold = leave_one_framework_out([flask, target_example], min_groups=1)[0] - assert fold.held_framework == "flask" and target_example.group in fold.eval_groups - demos = [{"example_id": flask.id, "alternate": {"example_id": alt.id}}] - usable = demonstrations_for(demos, by_id([flask, alt]), target, fold) - assert [d["example_id"] for d in usable] == [alt.id] - assert demonstrations_for([{"example_id": flask.id, "alternate": None}], by_id([flask]), target, fold) == [] - - def test_assert_boundary_catches_an_ineligible_rendered_id() -> None: examples = corpus(groups=6) fold = outer_folds(examples, k=3)[0] diff --git a/tests/test_learn_roles.py b/tests/test_learn_roles.py index 9068e4c..01e1045 100644 --- a/tests/test_learn_roles.py +++ b/tests/test_learn_roles.py @@ -7,7 +7,7 @@ from __future__ import annotations import json from pathlib import Path -from openultrasast.learn.features import Vocabulary, build_for_scan, function_of, model_sinks_part, roles_part +from openultrasast.learn.features import Vocabulary, function_of, model_sinks_part, roles_part from openultrasast.learn.roles import ( RoleSet, base_roles, @@ -168,22 +168,6 @@ def test_a_language_without_declarations_is_none_not_failed() -> None: assert source_part(["def g():", " pass"], "python", "missing").state == "failed" -def test_build_for_scan_records_the_roles_instrument(tmp_path: Path) -> None: - from openultrasast.findings import quick_scan_findings - from openultrasast.preprocess import preprocess_repository - from openultrasast.rank import rank_targets - - _write(tmp_path, "app/one.py", "import os\n\n\ndef inner(value):\n os.system(value)\n") - _, targets = preprocess_repository(tmp_path) - assert targets and (tmp_path / targets[0].path).stat().st_size > 0, "the fixture was read" - findings = quick_scan_findings(tmp_path, targets, rank_targets(targets)) - records = build_for_scan(tmp_path, findings, None) - assert records and records[0]["instruments"]["roles"]["state"] == "ran" - assert records[0]["x"]["roles.sink_confidence"] == 1.0 - off = build_for_scan(tmp_path, findings, None, roles=False) - assert off[0]["instruments"]["roles"]["state"] == "none" and off[0]["x"]["roles.sink_confidence"] is None - - # --- model roles: the plane task under a scripted client ---------------------------------------------------------- diff --git a/tests/test_learn_schema.py b/tests/test_learn_schema.py index fb69b72..9bc7126 100644 --- a/tests/test_learn_schema.py +++ b/tests/test_learn_schema.py @@ -10,14 +10,10 @@ from typing import Any import pytest -from openultrasast.findings import quick_scan_findings from openultrasast.learn import schema -from openultrasast.learn.features import Delta, Part, build_for_scan, record +from openultrasast.learn.features import Part, record from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record -from openultrasast.mapping import analyze_entry_points, attach_reachability_hints from openultrasast.plane.tasks.features import case_features -from openultrasast.preprocess import preprocess_repository -from openultrasast.rank import rank_targets ROOT = Path(__file__).resolve().parents[1] APP = """import os @@ -32,22 +28,6 @@ def {outer}(): """ -def _scan(root: Path) -> list[dict[str, Any]]: - _, targets = preprocess_repository(root) - assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read" - targets = attach_reachability_hints(targets, analyze_entry_points(root, targets)) - findings = quick_scan_findings(root, targets, rank_targets(targets)) - assert findings, "the fixture's os.system fires a quick rule" - engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []} - return build_for_scan(root, findings, engine) - - -def _repo(root: Path, path: str, inner: str, outer: str) -> Path: - (root / path).parent.mkdir(parents=True, exist_ok=True) - (root / path).write_text(APP.format(inner=inner, outer=outer)) - return root - - # --- the allow-list ---------------------------------------------------------------------------------------------- @@ -63,35 +43,10 @@ def test_features_are_allow_listed(tmp_path: Path) -> None: mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"] assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary" assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS) - records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta")) - assert records - for rec in records: - assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES" - assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"]) assert {s.name for s in schema.features_for("static")} < set(names) assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"] -def test_record_is_name_invariant(tmp_path: Path) -> None: - """Two repositories that differ only in repository name, paths, file and function names and commit ids give the - same features: nothing identifies a repository.""" - first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta")) - second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta")) - assert [r["x"] for r in first] == [r["x"] for r in second] - assert [r["instruments"] for r in first] == [r["instruments"] for r in second] - assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope" - assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None - root = tmp_path / "acme-shop" - _, targets = preprocess_repository(root) - findings = quick_scan_findings(root, targets, rank_targets(targets)) - changed = {"app/one.py": [(5, 5)]} - one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40)) - two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40)) - assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40 - assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1 - assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature" - - # --- label fields ------------------------------------------------------------------------------------------------ AGREED = { @@ -165,17 +120,6 @@ def test_missing_is_not_zero() -> None: assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero" -def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None: - root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta") - _, targets = preprocess_repository(root) - findings = quick_scan_findings(root, targets, rank_targets(targets)) - unread = build_for_scan(root, findings, {"questions": 0, "findings": []}) - absent = build_for_scan(root, findings, None) - assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"} - assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"} - assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent)) - - def test_validate_record_refuses_values_outside_their_type() -> None: good = _plane()[0] validate_record(good) diff --git a/tests/test_mapping.py b/tests/test_mapping.py index 8a7a0cc..8d828cd 100644 --- a/tests/test_mapping.py +++ b/tests/test_mapping.py @@ -4,13 +4,7 @@ from pathlib import Path from openultrasast.mapping import ( analyze_entry_points, attach_reachability_hints, - attach_static_hints, - codeql_mapping_tasks, - differential_mapping_record, ingest_sarif, - semgrep_mapping_tasks, - sharp_edge_record, - verifier_evidence_candidates, write_entry_points, write_static_hints, ) @@ -32,56 +26,6 @@ def test_ingest_sarif_normalizes_semgrep_hints(tmp_path: Path) -> None: assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval" -def test_static_hints_attach_to_file_targets_and_evidence_candidates(tmp_path: Path) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("eval(user_input)\n") - hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "py/code-injection", "app.py")) - _, targets = preprocess_repository(repo, static_hints=hints) - - enriched = attach_static_hints(targets, hints) - evidence = verifier_evidence_candidates(hints, path="app.py") - - assert enriched[0].static_hints[0]["analyzer"] == "codeql" - assert evidence == [ - { - "source": "codeql", - "rule_id": "py/code-injection", - "path": "app.py", - "line": 7, - "severity": "high", - "message": "Potential injection", - "provenance": "sarif:codeql:py/code-injection", - } - ] - - -def test_mapping_task_records_cover_disciplines(tmp_path: Path) -> None: - semgrep_hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "parser.c")) - codeql_hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "cpp/overflow", "parser.c")) - - assert semgrep_mapping_tasks(semgrep_hints)[0].task == "pattern_variant_review" - assert codeql_mapping_tasks(codeql_hints)[0].task == "source_sink_sanitizer_path_review" - assert ( - differential_mapping_record( - "parser.c", - change_kind="modified_function", - trust_boundary="network_input", - rationale="Parser changed behind a socket boundary.", - ).trust_boundary - == "network_input" - ) - assert ( - sharp_edge_record( - "config.py", - category="insecure_default", - surface="debug mode", - rationale="Debug default changes production risk.", - ).category - == "insecure_default" - ) - - def test_write_static_hints_artifact(tmp_path: Path) -> None: hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py")) output = tmp_path / "mapping" / "static_hints.json" diff --git a/tests/test_model_config_value.py b/tests/test_model_config_value.py index 25d64e4..21ffb86 100644 --- a/tests/test_model_config_value.py +++ b/tests/test_model_config_value.py @@ -26,17 +26,17 @@ def _cpg(rows): # type: ignore[no-untyped-def] def test_a_permissive_literal_is_entailed() -> None: - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts from openultrasast.model.ladder import Rung rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}] - answer = verdict(_cpg(rows), _spec(), function="create_app") + answer = verdicts(_cpg(rows), _spec(), function="create_app")[0] assert answer is not None and answer.rung is Rung.ENTAILED assert "*" in answer.witness and "CORS" in answer.witness def test_a_restrictive_literal_yields_no_finding() -> None: - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts rows = [ { @@ -47,12 +47,12 @@ def test_a_restrictive_literal_yields_no_finding() -> None: "args": ["app", '"https://app.example"'], } ] - assert verdict(_cpg(rows), _spec(), function="create_app") is None + assert verdicts(_cpg(rows), _spec(), function="create_app") == [] def test_a_non_literal_argument_is_corroborated_not_entailed() -> None: """The value is computed, so constant abstraction cannot decide it — that residual is the LLM's.""" - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts from openultrasast.model.ladder import Rung rows = [ @@ -64,30 +64,30 @@ def test_a_non_literal_argument_is_corroborated_not_entailed() -> None: "args": ["app", "configured_origins"], } ] - answer = verdict(_cpg(rows), _spec(), function="create_app") + answer = verdicts(_cpg(rows), _spec(), function="create_app")[0] assert answer is not None and answer.rung is Rung.CORROBORATED def test_a_setting_that_is_not_in_the_spec_is_ignored() -> None: - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts rows = [{"setting": 'print("*")', "line": "1", "method": "create_app", "literalArgs": ['"*"'], "args": ['"*"']}] - assert verdict(_cpg(rows), _spec(), function="create_app") is None + assert verdicts(_cpg(rows), _spec(), function="create_app") == [] def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None: from openultrasast.cpg.backend import CpgResult - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts - assert verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") is None + assert verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") == [] def test_the_verdict_is_deterministic() -> None: - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}] cpg = _cpg(rows) - assert verdict(cpg, _spec(), function="login") == verdict(cpg, _spec(), function="login") + assert verdicts(cpg, _spec(), function="login")[0] == verdicts(cpg, _spec(), function="login")[0] # --- framework coverage and the weak-algorithm arm -------------------------------------------------------- @@ -113,24 +113,24 @@ def test_the_settings_table_covers_the_frameworks_in_the_corpus() -> None: def test_a_weak_algorithm_is_entailed_by_its_literal() -> None: """CWE-327: the danger is the algorithm NAMED, not a permissive flag.""" - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts from openultrasast.model.ladder import Rung from openultrasast.model.specs import config_specs spec = config_specs(language="python")["config_secrets"] rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}] - answer = verdict(_cpg(rows), spec, function="digest") + answer = verdicts(_cpg(rows), spec, function="digest")[0] assert answer is not None and answer.rung is Rung.ENTAILED assert "md5" in answer.witness.lower() def test_a_strong_algorithm_yields_no_finding() -> None: - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts from openultrasast.model.specs import config_specs spec = config_specs(language="python")["config_secrets"] rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}] - assert verdict(_cpg(rows), spec, function="digest") is None + assert verdicts(_cpg(rows), spec, function="digest") == [] def _javascript_spec(): # type: ignore[no-untyped-def] @@ -142,7 +142,7 @@ def _javascript_spec(): # type: ignore[no-untyped-def] def test_disabling_template_escaping_is_entailed() -> None: """The setting NodeGoat's cross-site scripting turns off, claimed where the evidence is: the value.""" - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts from openultrasast.model.ladder import Rung rows = [ @@ -154,7 +154,7 @@ def test_disabling_template_escaping_is_entailed() -> None: "args": ["{ autoescape: false }"], } ] - answer = verdict(_cpg(rows), _javascript_spec(), function="<module>") + answer = verdicts(_cpg(rows), _javascript_spec(), function="<module>")[0] assert answer is not None and answer.rung is Rung.ENTAILED assert "swig.setDefaults" in answer.witness and "false" in answer.witness @@ -167,7 +167,7 @@ def test_enabling_template_escaping_is_not_a_finding() -> None: reported on exactly the code that repairs it -- the commented-out `autoescape: true` two lines below NodeGoat's defect. """ - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts rows = [ { @@ -178,12 +178,12 @@ def test_enabling_template_escaping_is_not_a_finding() -> None: "args": ["{ autoescape: true }"], } ] - assert verdict(_cpg(rows), _javascript_spec(), function="<module>") is None + assert verdicts(_cpg(rows), _javascript_spec(), function="<module>") == [] def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None: """The control for the other direction: scoping must not quiet the settings it was not about.""" - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts from openultrasast.model.ladder import Rung rows = [ @@ -195,7 +195,7 @@ def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None: "args": ["{ origin: true }"], } ] - answer = verdict(_cpg(rows), _javascript_spec(), function="createApp") + answer = verdicts(_cpg(rows), _javascript_spec(), function="createApp")[0] assert answer is not None and answer.rung is Rung.ENTAILED @@ -206,13 +206,13 @@ def test_the_first_setting_in_a_file_is_the_one_reported() -> None: spelling of its line numbers. NodeGoat has a permissive session at line 78 and template escaping disabled at line 135, and which one surfaced flipped on that comparison. """ - from openultrasast.model.config_value import verdict + from openultrasast.model.config_value import verdicts rows = [ {"setting": "swig.setDefaults({ autoescape: false })", "line": "135", "method": "m", "literalArgs": ["false"], "args": []}, {"setting": "cors({ origin: true })", "line": "78", "method": "m", "literalArgs": ["true"], "args": []}, ] - answer = verdict(_cpg(rows), _javascript_spec(), function="m") + answer = verdicts(_cpg(rows), _javascript_spec(), function="m")[0] assert answer is not None and "line 78" in answer.witness, answer.witness diff --git a/tests/test_model_report.py b/tests/test_model_report.py deleted file mode 100644 index 777f5dd..0000000 --- a/tests/test_model_report.py +++ /dev/null @@ -1,90 +0,0 @@ -"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1). - -Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a -real error in this project's history: - - per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the - hand-written development slice scored 83% - the overfitting gap published beside the headline, so the distance between the development slice and the - real-world one is never something a reader has to compute - the rung on every a finding's strength is what established it; a report that omits the rung invites the - finding reader to treat a suspicion as a result -""" - -from __future__ import annotations - -import pytest - - -def _slice(name, pairs, covered): # type: ignore[no-untyped-def] - from openultrasast.model.report import SliceRow - - return SliceRow(slice=name, pairs=pairs, covered=covered) - - -def test_the_report_carries_a_row_per_slice() -> None: - from openultrasast.model.report import render - - text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)]) - assert "vibe-py" in text and "vfc-js" in text - assert "30" in text and "17" in text - - -def test_a_bare_aggregate_is_refused() -> None: - """Req 11.1: no cross-slice number without the rows it was computed from.""" - from openultrasast.model.report import render - - with pytest.raises(ValueError, match="per-slice"): - render([]) - - -def test_the_overfitting_gap_is_published_beside_the_headline() -> None: - """The development slice minus the deciding real-world slice, stated rather than left to be computed.""" - from openultrasast.model.report import overfitting_gap, render - - rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)] - gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js") - assert gap == pytest.approx(0.80 - 0.20) - assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower() - assert "+60" in render(rows, development="vibe-py", deciding="vfc-js") - - -def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None: - """A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one.""" - from openultrasast.model.report import overfitting_gap - - assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None - - -def test_every_finding_shows_its_rung() -> None: - from openultrasast.findings import StaticFinding - from openultrasast.model.ladder import Rung - from openultrasast.model.report import render_findings - - finding = StaticFinding( - finding_id="r:app.py:1", - path="app.py", - title="t", - severity="high", - confidence="medium", - evidence_level="static_corroboration", - rationale="why", - line=1, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[], - ranking_priority=1.0, - rung=Rung.ENTAILED, - ) - text = render_findings([finding]) - assert "model_entailed" in text and "app.py" in text - - -def test_the_deciding_slice_is_named_in_the_report() -> None: - """Req 9.3: the real-world slice decides, and the report says which one that is.""" - from openultrasast.model.report import render - - text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js") - assert "vfc-js" in text and "decid" in text.lower() diff --git a/tests/test_model_taint.py b/tests/test_model_taint.py index 861e97f..aac6c44 100644 --- a/tests/test_model_taint.py +++ b/tests/test_model_taint.py @@ -30,12 +30,12 @@ def _spec(): # type: ignore[no-untyped-def] def test_an_unsanitized_source_to_sink_path_is_entailed() -> None: from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ {"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3} ] - answer = verdict(_cpg(rows), _spec(), function="run") + answer = verdicts(_cpg(rows), _spec(), function="run")[0] assert answer is not None and answer.rung is Rung.ENTAILED assert answer.family == "injection" assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found" @@ -44,30 +44,30 @@ def test_an_unsanitized_source_to_sink_path_is_entailed() -> None: def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None: """The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's.""" from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}] - answer = verdict(_cpg(rows), _spec(), function="run") + answer = verdicts(_cpg(rows), _spec(), function="run")[0] assert answer is not None and answer.rung is Rung.CORROBORATED def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None: - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts - assert verdict(_cpg([]), _spec(), function="run") is None + assert verdicts(_cpg([]), _spec(), function="run") == [] def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None: """`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3.""" from openultrasast.cpg.backend import CpgResult - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None) - assert verdict(broken, _spec(), function="run") is None + assert verdicts(broken, _spec(), function="run") == [] def test_the_verdict_is_restricted_to_the_labeled_function() -> None: - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -79,20 +79,20 @@ def test_the_verdict_is_restricted_to_the_labeled_function() -> None: "length": 2, } ] - assert verdict(_cpg(rows), _spec(), function="run") is None, "a flow in another function does not judge this one" + assert verdicts(_cpg(rows), _spec(), function="run") == [], "a flow in another function does not judge this one" def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None: """Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average.""" from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ {"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9}, {"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2}, ] - first = verdict(_cpg(rows), _spec(), function="run") - second = verdict(_cpg(rows), _spec(), function="run") + first = verdicts(_cpg(rows), _spec(), function="run")[0] + second = verdicts(_cpg(rows), _spec(), function="run")[0] assert first == second assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness @@ -100,7 +100,7 @@ def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() def test_the_spec_drives_the_query_parameters() -> None: """The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam.""" from openultrasast.cpg.backend import CpgResult - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts seen: dict[str, object] = {} @@ -109,7 +109,7 @@ def test_the_spec_drives_the_query_parameters() -> None: seen.update(params) return [] - verdict(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run") + verdicts(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run") assert seen["query"] == "taint" assert seen["sources"] == ("request.args",) assert seen["sinks"] == ("os.system", "execute") @@ -143,7 +143,7 @@ def _shape_spec(): # type: ignore[no-untyped-def] def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_reaches_it() -> None: - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -157,12 +157,12 @@ def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_r "sinkArg0Literal": True, } ] - assert verdict(_cpg(rows), _shape_spec(), function="run") is None, "a bound query is the fix, not the bug" + assert verdicts(_cpg(rows), _shape_spec(), function="run") == [], "a bound query is the fix, not the bug" def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None: from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -176,14 +176,14 @@ def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None: "sinkArg0Literal": False, } ] - answer = verdict(_cpg(rows), _shape_spec(), function="run") + answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0] assert answer is not None and answer.rung is Rung.ENTAILED def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> None: """`os.system(x)` with one argument is not "safe because it has one argument" -- the rule is per sink.""" from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -197,17 +197,17 @@ def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> N "sinkArg0Literal": False, } ] - answer = verdict(_cpg(rows), _spec(), function="run") # os.system is not in safe_shape_sinks + answer = verdicts(_cpg(rows), _spec(), function="run")[0] # os.system is not in safe_shape_sinks assert answer is not None and answer.rung is Rung.ENTAILED def test_a_row_without_shape_fields_is_judged_on_the_flow_alone() -> None: """Older rows, or an engine that could not report arity, must not be silently treated as safe.""" from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [{"sink": "db.execute(q)", "sinkLine": "5", "sinkMethod": "run", "source": "request.args['n']", "sanitized": False, "length": 3}] - answer = verdict(_cpg(rows), _shape_spec(), function="run") + answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0] assert answer is not None and answer.rung is Rung.ENTAILED @@ -219,7 +219,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None: So the flag must exist and must default to off. """ from openultrasast.cpg.backend import CpgResult - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts seen: dict[str, object] = {} @@ -227,9 +227,9 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None: seen.update(params) return [] - verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run") + verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run") assert seen["parameterSources"] == "false", "treating every parameter as untrusted is not the default" - verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True) + verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True) assert seen["parameterSources"] == "true" @@ -250,7 +250,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None: def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None: from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -263,13 +263,13 @@ def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None "inLabeledScope": True, } ] - answer = verdict(_cpg(rows), _spec(), function="requestListener") + answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0] assert answer is not None and answer.rung is Rung.ENTAILED assert "<lambda>0" not in answer.witness, "the witness names the flow, not the synthetic closure name" def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None: - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -282,17 +282,17 @@ def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None: "inLabeledScope": False, } ] - assert verdict(_cpg(rows), _spec(), function="requestListener") is None + assert verdicts(_cpg(rows), _spec(), function="requestListener") == [] def test_rows_without_the_scope_flag_fall_back_to_the_method_name() -> None: """A row from an engine that cannot report scope must not silently become in-scope.""" - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ {"sink": "os.system(x)", "sinkLine": "9", "sinkMethod": "<lambda>0", "source": "request.args['x']", "sanitized": False, "length": 2} ] - assert verdict(_cpg(rows), _spec(), function="run") is None + assert verdicts(_cpg(rows), _spec(), function="run") == [] def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter() -> None: @@ -305,7 +305,7 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter( breaks ties, so the order stays total and the verdict deterministic. """ from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -336,10 +336,10 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter( "inLabeledScope": True, }, ] - answer = verdict(_cpg(rows), _spec(), function="requestListener") + answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0] assert answer is not None and answer.rung is Rung.ENTAILED assert "request.args['f']" in answer.witness, f"witness named no real source: {answer.witness}" - assert verdict(_cpg(rows), _spec(), function="requestListener") == answer # still deterministic + assert verdicts(_cpg(rows), _spec(), function="requestListener")[0] == answer # still deterministic def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitized() -> None: @@ -351,7 +351,7 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize modelled source reaches the sink, those flows are the evidence and the bare-parameter ones are ignored. """ from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -373,14 +373,14 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize "inLabeledScope": True, }, ] - answer = verdict(_cpg(rows), _spec(), function="requestListener") + answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0] assert answer is not None and answer.rung is Rung.CORROBORATED, "a sanitized real flow is not an entailment" assert "request.args['f']" in answer.witness def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() -> None: from openultrasast.model.ladder import Rung - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts rows = [ { @@ -393,7 +393,7 @@ def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() "inLabeledScope": True, } ] - answer = verdict(_cpg(rows), _spec(), function="run") + answer = verdicts(_cpg(rows), _spec(), function="run")[0] assert answer is not None and answer.rung is Rung.ENTAILED @@ -443,13 +443,13 @@ def test_a_bound_that_governs_the_sink_lowers_the_rung() -> None: from openultrasast.cpg.backend import CpgResult from openultrasast.model.ladder import Rung from openultrasast.model.specs import taint_specs - from openultrasast.model.taint import verdict + from openultrasast.model.taint import verdicts spec = taint_specs(language="c")["memory"] assert "strcpy" in spec.bounded_sinks, "a CWE-121 sink is dischargeable by a bound" - governed = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main") - ungoverned = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main") + governed = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main")[0] + ungoverned = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main")[0] assert governed is not None and governed.rung is Rung.CORROBORATED assert "governed by" in governed.witness, "the witness names the bound it found" @@ -471,7 +471,7 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None: verdict, and `system` won it on flow length, so the SQL injection went unreported by construction.""" from openultrasast.cpg.backend import CpgResult from openultrasast.model.specs import taint_specs - from openultrasast.model.taint import verdict, verdicts + from openultrasast.model.taint import verdicts def row(sink, line, length): # type: ignore[no-untyped-def] return { @@ -497,5 +497,3 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None: assert len(every) == 2, "two sink sites are two findings" assert {v.location for v in every} == {"app.php:6:handler", "app.php:13:handler"} - # The pair path is unchanged: the strongest is still what `verdict` returns. - assert every[0].location == verdict(cpg, spec, function="", file="app.php").location diff --git a/tests/test_model_taxonomy.py b/tests/test_model_taxonomy.py index d0a5a17..45293f5 100644 --- a/tests/test_model_taxonomy.py +++ b/tests/test_model_taxonomy.py @@ -62,22 +62,6 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None assert taxonomy.family_of_cwe("CWE-99999") is None -def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None: - from openultrasast.model.taxonomy import load_families - - taxonomy = load_families() - assert taxonomy.related("injection", "injection") == "same" - assert taxonomy.related("injection", "access_control") == "lateral" - assert taxonomy.related("injection", "not_a_family") == "fabricated" - assert taxonomy.related("unknown", "injection") == "lateral" # abstention earns no partial credit - assert all(family.parent is None for family in taxonomy.families) # the shipped taxonomy is flat - (tmp_path / "families.toml").write_text( - GOOD + '\n[[family]]\nid = "path"\nparent = "injection"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "canary"\n' - ) - scoped = load_families(tmp_path / "families.toml", require_all=False) - assert scoped.related("path", "injection") == "parent_child" and scoped.related("injection", "path") == "parent_child" - - @pytest.mark.parametrize( ("text", "needle"), [ diff --git a/tests/test_plane_delivery.py b/tests/test_plane_delivery.py index 507c4ba..66e61c0 100644 --- a/tests/test_plane_delivery.py +++ b/tests/test_plane_delivery.py @@ -129,16 +129,6 @@ def test_state_is_mirrored_and_seeds_a_rerun_elsewhere(store: HttpObjectStore, t assert second.load_state() is None -def test_put_input_stores_a_file_no_task_produces(store: HttpObjectStore, tmp_path: Path) -> None: - delivery = Delivery(store, "r", tmp_path / "r") - source = tmp_path / "candidates.json" - source.write_bytes(b"[1, 2, 3]") - url = delivery.put_input("candidates.json", source) - assert store.objects["runs/r/inputs/candidates.json"] == b"[1, 2, 3]" - with urllib.request.urlopen(url, timeout=10) as response: # noqa: S310 - assert response.read() == b"[1, 2, 3]" - - def test_declared_artifacts_are_outputs_plus_what_consumers_name(tmp_path: Path) -> None: """A consumer may declare a producer's summary.json (every task writes one) without the producer listing it.""" from openultrasast.plane.manifests import load_manifests diff --git a/tests/test_plane_egress.py b/tests/test_plane_egress.py index c369967..1683535 100644 --- a/tests/test_plane_egress.py +++ b/tests/test_plane_egress.py @@ -3,8 +3,7 @@ Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the memory store's host (where the runner PUTs its tar and GETs its inputs), the bound Workspaces' Git hosts and the bound Model's API host, all as one TLS passthrough rule on 443 (plane-on-kubernetes 2.3), and nothing a gateway would reject or that -would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule (the receiver path, kept for -group 2 only, is the one exception). +would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule. """ from __future__ import annotations @@ -12,12 +11,11 @@ from __future__ import annotations import ipaddress import json import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import pytest -from openultrasast.plane import egress, runner +from openultrasast.plane import egress from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest MANIFESTS = """ @@ -63,7 +61,7 @@ def hostnames(policy: dict) -> list[str]: def assert_tight(policy: dict) -> None: for rule in policy["rules"]: - assert set(rule) <= {"http", "tlsPassthrough"}, f"only http and passthrough rules, got {rule}" + assert set(rule) == {"tlsPassthrough"}, f"only TLS passthrough rules, got {rule}" body = next(iter(rule.values())) assert "all" not in body["ports"], "never every port" for host in hostnames(policy): @@ -82,12 +80,6 @@ def test_model_free_task_gets_only_the_store(manifests) -> None: assert egress.policy_for(manifests.tasks["facts"], [], None, None) == {"rules": []}, "no store, no Git, no Model: nothing" -def test_the_receiver_path_adds_the_one_http_rule_until_it_is_removed(manifests) -> None: - policy = egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host=egress.RECEIVER_HOST) - assert policy == {"rules": [{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}}]} - assert egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host="127.0.0.1") == {"rules": []} - - def test_model_bound_task_with_git_workspaces(manifests) -> None: task = manifests.tasks["verify"] policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], STORE) @@ -191,50 +183,3 @@ def test_apply_waits_for_the_actor_then_gives_up(tmp_path: Path, kubectl_ate: Pa timer.start() cli.apply("late", "default", POLICY, timeout=10) timer.join() - - -def test_receiver_address_modes(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") - assert egress.receiver_address(4242, None) == ("http://127.0.0.1:4242/", None) - monkeypatch.delenv("OUSAST_ARTIFACT_HOST") - monkeypatch.setenv("OUSAST_ARTIFACT_DIAL", "172.19.0.1:80") - assert egress.receiver_address(18090, "kind-test") == (f"http://{egress.RECEIVER_HOST}/", "172.19.0.1:80") - monkeypatch.delenv("OUSAST_ARTIFACT_DIAL") - with pytest.raises(RuntimeError, match="kube context"): - egress.receiver_address(18090, None) # no context to look the Service up in: never a built one - - -# --- the runner dials the gateway address with the receiver's name as Host ---------------------------------------- - - -class _Seen(ThreadingHTTPServer): - daemon_threads = True - seen: list[tuple[str, str | None, bytes]] - - -class _Handler(BaseHTTPRequestHandler): - server: _Seen - - def do_POST(self) -> None: # noqa: N802 - body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) - self.server.seen.append((self.path, self.headers.get("Host"), body)) - self.send_response(200) - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, format: str, *args: object) -> None: # noqa: A002 - pass - - -def test_delivery_dials_the_override_with_the_url_host() -> None: - server = _Seen(("127.0.0.1", 0), _Handler) - server.seen = [] - threading.Thread(target=server.serve_forever, daemon=True).start() - try: - dial = f"127.0.0.1:{server.server_address[1]}" - url = f"http://{egress.RECEIVER_HOST}/in" - assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, dial=dial) == 200 - assert server.seen == [("/in", egress.RECEIVER_HOST, b"tar")] - finally: - server.shutdown() - server.server_close() diff --git a/tests/test_plane_features.py b/tests/test_plane_features.py index facad63..439e7f1 100644 --- a/tests/test_plane_features.py +++ b/tests/test_plane_features.py @@ -9,10 +9,9 @@ from pathlib import Path import pytest -from openultrasast.learn.schema import FeatureRecordError, feature_set_digest from openultrasast.plane.memory import FileStore from openultrasast.plane.tasks import features -from openultrasast.plane.tasks.remember import Context, remember_run, rows_for +from openultrasast.plane.tasks.remember import remember_run FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46" CASE = "lmdeploy-media-url-ssrf" @@ -25,30 +24,6 @@ def run_dir(tmp_path: Path) -> Path: return target -def _checkout(root: Path) -> Path: - media = root / "lmdeploy" / "vl" / "media" - media.mkdir(parents=True) - (media / "connection.py").write_text("def _is_safe_url(u):\n return True\n\n\ndef _load_http_url(u):\n x = 1\n return x\n") - return root - - -def test_features_of_the_recorded_case(run_dir: Path, tmp_path: Path) -> None: - [rec] = features.recorded_case(run_dir, CASE, workspace=_checkout(tmp_path / "checkout")) - assert (rec["candidate"], rec["family"], rec["profile"], rec["unit"]) == ( - "lmdeploy/vl/media/connection.py::_load_http_url", "untrusted_destination", "plane", "pin", - ) # fmt: skip - x, states = rec["x"], {k: v["state"] for k, v in rec["instruments"].items()} - assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, True, None, 2) - assert x["agree.final"] == "agreed" and x["facts.callers"] == 0 and x["facts.is_global"] is False - assert x["facts.function_lines"] == 3 and x["lang.python"] is True and x["verify.site_offset"] == 0 - assert states["verify"] == states["agree"] == states["facts"] == states["source"] == "ran" - assert states["quick"] == states["engine"] == "none", "no alerts task in this Run: no coverage, not zero hits" - assert rec["instruments"]["verify"]["version"] == "deepseek-flash" and rec["feature_set_digest"] == feature_set_digest() - assert "site_match" not in json.dumps(rec), "the fixture's site_match: true was dropped on read" - [without] = features.recorded_case(run_dir, CASE) - assert without["instruments"]["source"]["state"] == "none" and without["x"]["facts.function_lines"] is None - - def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None: out = tmp_path / "out" env = { @@ -71,13 +46,5 @@ def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_pat assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed" -def test_remember_refuses_a_record_outside_the_allow_list(run_dir: Path) -> None: - [rec] = features.recorded_case(run_dir, CASE) - rec["x"]["site_match"] = True - ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64) - with pytest.raises(FeatureRecordError, match="site_match"): - rows_for(ctx, facts=None, passes={}, models={}, agreed=None, features=[rec]) - - def test_the_task_fails_loudly_without_an_output_dir() -> None: assert features.main({}) == 2 diff --git a/tests/test_plane_increment.py b/tests/test_plane_increment.py index 46b39b1..613b688 100644 --- a/tests/test_plane_increment.py +++ b/tests/test_plane_increment.py @@ -77,7 +77,7 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None: assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls env = {e.name: e.value for e in task.env} assert env["OUSAST_PASS"] == entry.name[-1] - docs = reconciler.render_task(run, entry, manifests, "http://127.0.0.1:1/") + docs = reconciler.render_task(run, entry, manifests) for doc in docs: assert not non_ax_fields(doc["spec"], AX_FIELDS[doc["kind"]], doc["kind"]) assert set(doc["metadata"]) <= {"name", "atespace"} diff --git a/tests/test_plane_reconciler.py b/tests/test_plane_reconciler.py index 9375a8a..5e35c1d 100644 --- a/tests/test_plane_reconciler.py +++ b/tests/test_plane_reconciler.py @@ -5,7 +5,7 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase -- and on the scripted poll it PUTs a scripted tar to the presigned ``delivery.put`` URL the start request carried -(plane-on-kubernetes 2.4; or POSTs it to ``OUSAST_ARTIFACT_URL`` in receiver mode), or turns ``Failed`` with a +(plane-on-kubernetes 2.4), or turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery, and a delivery happens only after the reconciler's start request reached the fake router (Req 4.5). The store is ``plane_fake_store.HttpObjectStore``. """ @@ -21,8 +21,6 @@ import sys import tarfile import threading import time -import urllib.error -import urllib.request from collections.abc import Iterator from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -152,11 +150,9 @@ elif argv[:2] == ["get", "task"]: delivery = body.get("delivery") or {{}} if script.get("deliver", True) and "phase" not in script and not script.get("vanish"): files = script.get("files", {{}}) - base = env.get("OUSAST_ARTIFACT_URL", "").rstrip("/") + "/inputs/" for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items(): presigned = (delivery.get("inputs") or {{}}).get(name) - fetch = urllib.request.Request(presigned) if presigned else urllib.request.Request( - base + ref, headers={{"X-Ousast-Run": env["OUSAST_RUN"], "X-Ousast-Task": task}}) + fetch = urllib.request.Request(presigned) try: with urllib.request.urlopen(fetch) as resp: dest = HERE / "fetched" / task / ref @@ -175,13 +171,8 @@ elif argv[:2] == ["get", "task"]: info = tarfile.TarInfo(path) info.size = len(data) tar.addfile(info, io.BytesIO(data)) - headers = {{"Content-Type": "application/x-tar", "X-Ousast-Run": env["OUSAST_RUN"]}} - headers["X-Ousast-Task"] = script.get("as_task", task) - if delivery.get("put"): # the store path: PUT to the presigned URL, no receiver headers - tar_headers = {{"Content-Type": "application/x-tar"}} - req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers) - else: - req = urllib.request.Request(env["OUSAST_ARTIFACT_URL"], data=buf.getvalue(), method="POST", headers=headers) + tar_headers = {{"Content-Type": "application/x-tar"}} + req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers) try: with urllib.request.urlopen(req) as resp: (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n") @@ -420,7 +411,6 @@ def test_chain_runs_in_order_with_inputs_env_and_model(fake: Fake, tmp_path: Pat assert env["OUSAST_BUDGET_USD"] == "2.5" and env["OUSAST_BUDGET_CALLS"] == "40" assert env["OUSAST_INPUT_FACTS"] == "/workspace/.ousast-in/verify/repo-facts/facts.json" assert json.loads(env["OUSAST_INPUTS"]) == {"FACTS": "repo-facts/facts.json"} - assert "OUSAST_ARTIFACT_URL" not in env, "the store path renders no receiver address into the Task" assert env["OUSAST_MODEL"] == "deepseek-flash" assert json.loads(env["OUSAST_MODEL_PARAMS"]) == {"cache_hit_per_m": 0.07, "input_per_m": 0.27, "output_per_m": 1.1} kinds = [(d["kind"], d["metadata"]["name"]) for d in verify["docs"]] @@ -661,33 +651,6 @@ def tar_of(files: dict[str, bytes]) -> bytes: return buf.getvalue() -def test_receiver_rejects_traversal_unknown_tasks_and_wrong_types(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") # the direct test address: no Service lookup, no context needed - receiver = reconciler.Receiver("r", tmp_path / "r", 0) - receiver.running.add("t") - headers = {"X-Ousast-Run": "r", "X-Ousast-Task": "t", "Content-Type": "application/x-tar"} - code, text = receiver.accept(headers, tar_of({"../escape.json": b"{}"})) - assert code == 400 and "escape" in text and not (tmp_path / "escape.json").exists() - assert receiver.accept({**headers, "X-Ousast-Task": "other"}, tar_of({"summary.json": b"{}"}))[0] == 403 - assert receiver.accept({**headers, "X-Ousast-Run": "someone"}, tar_of({"summary.json": b"{}"}))[0] == 403 - assert receiver.accept({**headers, "Content-Type": "application/json"}, b"{}")[0] == 415 - assert receiver.accept(headers, b"not a tar")[0] == 400 - assert receiver.accept(headers, tar_of({"summary.json": b"{}", "sub/x.json": b"1"}))[0] == 200 - assert (tmp_path / "r" / "t" / "sub" / "x.json").read_text() == "1" and "t" in receiver.delivered - receiver.server_close() - - -def test_delivery_for_a_task_that_is_not_running_is_rejected(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("OUSAST_DELIVERY", "receiver") # the receiver path, kept for group 2 (task 2.6 removes this test) - monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "1") # the rejected delivery is no delivery: the task times out - script = json.loads(json.dumps(SCRIPT)) - script["repo-facts"]["as_task"] = "agree" # the runner claims to be a task that has not started - fake.script(script) - assert reconciler.run(write_run(tmp_path, "spoof"), ax=str(fake.ax)) == "failed" - assert [n for _, n in fake.events("rejected")] == ["repo-facts"] - assert not (reconciler.run_dir("spoof") / "agree" / "summary.json").exists() - - def test_lock_refuses_a_second_runner_and_ignores_a_dead_one(fake: Fake, tmp_path: Path) -> None: manifest = write_run(tmp_path, "locked") base = reconciler.run_dir("locked") @@ -824,7 +787,7 @@ def test_rendered_documents_carry_only_ax_fields(tmp_path: Path) -> None: (tmp_path / "repo-facts").mkdir() (tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8") run, manifests = reconciler.load_run(write_run(tmp_path, "fields")) - docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests, "http://h:1/")] + docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests)] assert {d["kind"] for d in docs} == {"Task", "Workspace", "Model"} for rendered in docs: where = f"{rendered['kind']}/{rendered['metadata']['name']}" @@ -841,7 +804,7 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None: (tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8") run, manifests = reconciler.load_run(write_run(tmp_path, "order")) for entry in run.tasks: - docs = reconciler.render_task(run, entry, manifests, "http://h:1/") + docs = reconciler.render_task(run, entry, manifests) assert [d["kind"] for d in docs].count("Task") == 1 assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs] @@ -943,44 +906,6 @@ def test_a_refused_egress_policy_fails_the_task_before_resume(fake: Fake, tmp_pa SUBSTRATE_ENV_LIMIT = 32768 # "actor_template.containers[0].env[11].value: Too long" on the live cluster, 2026-09-29 -def get_input(receiver: reconciler.Receiver, path: str, run: str = "r", task: str = "t") -> tuple[int, bytes]: - url = f"http://127.0.0.1:{receiver.server_address[1]}{path}" - request = urllib.request.Request(url, headers={"X-Ousast-Run": run, "X-Ousast-Task": task}) - try: - with urllib.request.urlopen(request, timeout=10) as response: - return response.status, response.read() - except urllib.error.HTTPError as exc: - return exc.code, exc.read() - - -def test_receiver_serves_a_running_task_its_declared_inputs_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") - base = tmp_path / "r" - (base / "p").mkdir(parents=True) - big = b"x" * (3 << 20) - (base / "p" / "a.json").write_bytes(big) - (base / "p" / "b.json").write_bytes(b"not declared") - (tmp_path / "secret.txt").write_text("outside the run") - receiver = reconciler.Receiver("r", base, 0) - threading.Thread(target=receiver.serve_forever, daemon=True).start() - try: - receiver.running.update({"t", "idle"}) - receiver.inputs.update({"t": frozenset({"p/a.json", "p/missing.json", "../secret.txt"}), "done": frozenset({"p/a.json"})}) - assert get_input(receiver, "/inputs/p/a.json") == (200, big), "streamed whole" - assert get_input(receiver, "/inputs/p/a.json", run="other")[0] == 403, "wrong run" - assert get_input(receiver, "/inputs/p/a.json", task="done")[0] == 403, "a task that is not running" - assert get_input(receiver, "/inputs/p/a.json", task="idle")[0] == 403, "a running task without that input" - assert get_input(receiver, "/inputs/p/b.json")[0] == 403, "not among the task's inputs" - assert get_input(receiver, "/inputs/p/missing.json")[0] == 404, "declared but not delivered" - for escape in ("/inputs/../secret.txt", "/inputs/%2e%2e/secret.txt", "/inputs/p/../../secret.txt", "/inputs//etc/passwd"): - code, body = get_input(receiver, escape) - assert code in (403, 404) and b"outside" not in body, escape - assert get_input(receiver, "/p/a.json")[0] == 404, "only /inputs/ is served" - finally: - receiver.shutdown() - receiver.server_close() - - def test_a_missing_producer_artifact_fails_the_consumer_instead_of_running_it(fake: Fake, tmp_path: Path) -> None: script = json.loads(json.dumps(SCRIPT)) del script["repo-facts"]["files"]["facts.json"] # repo-facts reports done but never delivered facts.json @@ -996,7 +921,7 @@ def test_a_megabyte_producer_artifact_keeps_every_rendered_value_under_substrate (tmp_path / "repo-facts" / "facts.json").write_text(json.dumps({"blob": "f" * (1 << 20)}), encoding="utf-8") run, manifests = reconciler.load_run(write_run(tmp_path, "big")) for entry in run.tasks: - docs = reconciler.render_task(run, entry, manifests, "http://h:1/", "10.0.0.1:80") + docs = reconciler.render_task(run, entry, manifests) task, workspaces = docs[-1], [d for d in docs if d["kind"] == "Workspace"] assert all(len(e["value"]) < SUBSTRATE_ENV_LIMIT for e in task["spec"]["env"]), entry.name assert len(yaml.safe_dump(task)) < SUBSTRATE_ENV_LIMIT, f"{entry.name}: AX_TASK_YAML" @@ -1080,15 +1005,3 @@ def test_a_tar_the_store_path_cannot_accept_fails_the_task(fake: Fake, tmp_path: assert not (reconciler.run_dir("esc").parent / "escape.json").exists() -def test_receiver_mode_still_runs_through_the_host_receiver(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """OUSAST_DELIVERY=receiver (group 2 only): the old path, with its http rule and OUSAST_ARTIFACT_URL, until 2.6.""" - monkeypatch.setenv("OUSAST_DELIVERY", "receiver") - assert reconciler.run(write_run(tmp_path, "rcv"), ax=str(fake.ax)) == "done" - assert fake.applied("rcv", "verify")["env"]["OUSAST_ARTIFACT_URL"].startswith("http://127.0.0.1:") - assert [s["body"].get("delivery") for s in fake.router.starts] == [None, None, None], "no presigned URLs in receiver mode" - assert not any(k.startswith("runs/rcv/") and k.endswith("output.tar") for k in fake.store.objects), "nothing PUT to the store" - assert "runs/rcv/state.json" in fake.store.objects, "the state is still mirrored" - written = json.loads((tmp_path / "policies" / "default_rcv-repo-facts.written.json").read_text()) - assert written["rules"] == [{"tlsPassthrough": {"hostnames": ["example.invalid"], "ports": {"numbers": [443]}}}], ( - "an IP receiver: no http rule" - ) diff --git a/tests/test_plane_remember.py b/tests/test_plane_remember.py index 3c8a8cc..cacdf9c 100644 --- a/tests/test_plane_remember.py +++ b/tests/test_plane_remember.py @@ -39,7 +39,7 @@ def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]: return rows -def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None: +def test_remember_turns_the_saved_case_into_rows(run_dir: Path, tmp_path: Path) -> None: facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes() assert len(facts) > 100, "the fixture was read" store = FileStore(tmp_path / "memory") diff --git a/tests/test_plane_runner.py b/tests/test_plane_runner.py index 366dc62..9804f47 100644 --- a/tests/test_plane_runner.py +++ b/tests/test_plane_runner.py @@ -19,13 +19,12 @@ import time import urllib.error import urllib.request from collections.abc import Iterator -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import pytest from plane_fake_store import HttpObjectStore -from openultrasast.plane import egress, runner +from openultrasast.plane import runner from openultrasast.plane.manifests import GitSource from openultrasast.plane.router import Router, StartError from openultrasast.plane.runner import ( @@ -194,96 +193,10 @@ def test_main_serves_readiness_on_a_free_port_and_the_task_sees_200( # --- artifact delivery (Requirement 4.4) -------------------------------------------------------------------------------- -class Receiver(ThreadingHTTPServer): - """Records every POST (headers and body) and answers with ``status``.""" - - def __init__(self, status: int) -> None: - super().__init__(("127.0.0.1", 0), _ReceiverHandler) - self.status = status - self.posts: list[tuple[dict[str, str], bytes]] = [] - self.thread = threading.Thread(target=self.serve_forever, daemon=True) - self.thread.start() - - @property - def url(self) -> str: - return f"http://127.0.0.1:{self.server_address[1]}/artifacts" - - def stop(self) -> None: - self.shutdown() - self.server_close() - - -class _ReceiverHandler(BaseHTTPRequestHandler): - server: Receiver - - def do_POST(self) -> None: # noqa: N802 - body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) - self.server.posts.append(({k: v for k, v in self.headers.items()}, body)) - self.send_response(self.server.status) - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, format: str, *args: object) -> None: # noqa: A002 - pass - - -@pytest.fixture -def receiver() -> Iterator[Receiver]: - server = Receiver(200) - try: - yield server - finally: - server.stop() - - -def test_output_dir_is_delivered_as_a_tar_with_the_headers( - env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, OUSAST_RUN="validation-46") - assert code == 0 - assert len(receiver.posts) == 1 - headers, body = receiver.posts[0] - assert headers["Content-Type"] == "application/x-tar" - assert headers["X-Ousast-Task"] == "stub" - assert headers["X-Ousast-Run"] == "validation-46" - receiver.posts.clear() - again = {"OUSAST_ARTIFACT_URL": receiver.url, "OUSAST_TASK": "facts", "OUSAST_STATE_DIR": str(tmp_path / "again-state")} - code, _ = run_main(monkeypatch, tmp_path / "again", files_workspace_yaml(), **again) - assert code == 0 and receiver.posts[0][0]["X-Ousast-Task"] == "facts", "the Run's task name, not the ax Task's" - with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as archive: - names = sorted(archive.getnames()) - summary = archive.extractfile("summary.json") - assert summary is not None - assert json.loads(summary.read())["status"] == "done" - assert names == ["facts.json", "summary.json"] - - -def test_run_header_is_absent_without_ousast_run( - env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url) - assert code == 0 - assert "X-Ousast-Run" not in receiver.posts[0][0] - - -def test_delivery_failure_is_reported_as_failed_with_exit_2(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - broken = Receiver(500) - try: - code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url) - finally: - broken.stop() - assert code == EXIT_BY_STATUS["failed"] == 2 - assert len(broken.posts) == 3, "three attempts" - summary = json.loads((output / "summary.json").read_text()) - assert summary["status"] == "failed" - assert summary["reason"].startswith("artifact delivery failed: HTTP 500") - assert summary["units_done"] == 1 and summary["units_total"] == 1, "the task's other fields are kept" - - def test_deliver_retries_on_connection_errors_then_raises() -> None: - closed = Receiver(200) - url = closed.url - closed.stop() + closed = HttpObjectStore() + url = closed.presign_put("output.tar") + closed.close() with pytest.raises(DeliveryError, match="3 attempts"): deliver(url, b"x", {"Content-Type": "application/x-tar"}, backoff=0) @@ -366,9 +279,9 @@ def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name: return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log) -def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: +def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], tmp_path: Path) -> None: port = free_port() - proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url) + proc = spawn_runner(tmp_path, port) marker = tmp_path / "state" / "stub.done.json" try: wait_for(marker.exists) @@ -377,8 +290,7 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: base = f"http://127.0.0.1:{port}" assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200 assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200 - assert len(receiver.posts) == 1 - assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": True, "summary": None} + assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": False, "summary": None} proc.send_signal(signal.SIGTERM) assert proc.wait(timeout=10) == 0 finally: @@ -386,9 +298,9 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: proc.wait() -def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: +def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], tmp_path: Path) -> None: port = free_port() - proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="sleep", OUSAST_TERM_GRACE="1") + proc = spawn_runner(tmp_path, port, STUB_MODE="sleep", OUSAST_TERM_GRACE="1") pids_file = tmp_path / "out" / "pids.json" try: wait_for(pids_file.exists) @@ -403,45 +315,21 @@ def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: di proc.kill() proc.wait() assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete" - assert receiver.posts == [], "nothing is delivered for an interrupted command" def test_the_completion_marker_prevents_a_rerun_on_resume( - env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: counter = tmp_path / "runs.txt" - code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) - assert code == 0 and len(counter.read_text().splitlines()) == 1 and len(receiver.posts) == 1 + code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter)) + assert code == 0 and len(counter.read_text().splitlines()) == 1 (tmp_path / "ws" / "candidates.json").write_text("changed by the task\n") - code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) + code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter)) assert code == 0 assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work" - assert len(receiver.posts) == 1, "a delivered result is not delivered again" assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once" -def test_a_failed_delivery_is_retried_alone_on_the_next_boot( - env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - counter = tmp_path / "runs.txt" - broken = Receiver(500) - try: - code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url, STUB_COUNTER=str(counter)) - finally: - broken.stop() - assert code == 2 - marker = json.loads((tmp_path / "state" / "stub.done.json").read_text()) - assert marker["delivered"] is False and marker["status"] == "failed" and marker["exit"] == 0 - code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) - assert code == 0 - assert len(counter.read_text().splitlines()) == 1, "only the delivery is retried" - with tarfile.open(fileobj=io.BytesIO(receiver.posts[0][1]), mode="r:") as archive: - summary = archive.extractfile("summary.json") - assert summary is not None and json.loads(summary.read())["status"] == "done", "the task's own summary" - marker = json.loads((tmp_path / "state" / "stub.done.json").read_text()) - assert marker["delivered"] is True and marker["status"] == "done" - - # --- git workspaces from the case cache and by shallow clone ---------------------------------------------------------------- @@ -590,14 +478,13 @@ def test_git_commits_annotation_round_trips_to_the_checkout( "apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata: {name: rt}\nspec: {tasks: [{name: stub, task: stub}]}\n" ) run, manifests = reconciler.load_run(manifest) - *rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests, "unused") + *rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests) assert rest[0]["metadata"] == {"name": "case"}, "annotations are stripped before ax" assert rest[0]["spec"]["git"] == [{"name": "code", "repo": repo.as_uri(), "branch": "main"}], "no commit for ax" overrides = {"OUSAST_OUTPUT_DIR": str(output)} task_doc["spec"]["env"] = [ {"name": e["name"], "value": overrides.get(e["name"], e["value"])} for e in task_doc["spec"]["env"] - if e["name"] != "OUSAST_ARTIFACT_URL" ] assert json.loads({e["name"]: e["value"] for e in task_doc["spec"]["env"]}["OUSAST_GIT_PINS"]) == {"case/code": pin} monkeypatch.setenv("AX_TASK_YAML", yaml.safe_dump(task_doc)) @@ -669,11 +556,11 @@ def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child( - env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture + env: dict[str, str], tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: caplog.set_level(logging.DEBUG) counter = tmp_path / "runs.txt" - thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) + thread, environ, codes = main_in_thread(tmp_path, STUB_COUNTER=str(counter)) port = environ["AX_RUNNER_BOUND_PORT"] time.sleep(0.5) assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request" @@ -693,15 +580,14 @@ def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there" assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files" assert files_containing(tmp_path, SECRET) == [], "no file holds the credential" - assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar" assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record" assert any("start request: 202" in record.getMessage() for record in caplog.records) def test_a_crash_that_prints_the_credential_is_redacted( - env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str] + env: dict[str, str], tmp_path: Path, capfd: pytest.CaptureFixture[str] ) -> None: - thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak") + thread, environ, codes = main_in_thread(tmp_path, STUB_MODE="leak") start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}} assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202 thread.join(timeout=30) @@ -727,23 +613,25 @@ def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch: assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists() -def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: +def test_the_golden_boot_never_runs_the_command(env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path) -> None: """Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start.""" counter = tmp_path / "runs.txt" - same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"} + same = {"STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1", "AX_RUNNER_INSECURE_DELIVERY": "1"} real_port, golden_port = free_port(), free_port() real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same) golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same) try: for port in (real_port, golden_port): wait_for(lambda port=port: ready_on(port)) - Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}) + Router(f"http://127.0.0.1:{real_port}").start_task( + "default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}, {"put": object_store.presign_put("output.tar")} + ) marker = tmp_path / "state" / "stub.done.json" wait_for(marker.exists) time.sleep(1.0) assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command" - assert golden.poll() is None and len(receiver.posts) == 1 + assert golden.poll() is None with pytest.raises(StartError, match="already started"): Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {}) again_port = free_port() @@ -762,10 +650,10 @@ def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: R assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential" -def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: +def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], tmp_path: Path) -> None: """Substrate's golden boot never gets a start, so it must touch no workspace (and no network): a clone at boot failed on the live cluster while the actor was still being restored (2026-09-29).""" - thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url) + thread, environ, codes = main_in_thread(tmp_path) time.sleep(0.5) written = [p for p in (tmp_path / "ws").rglob("*") if p.is_file()] if (tmp_path / "ws").exists() else [] assert written == [], "nothing is materialised before the start request" @@ -778,82 +666,6 @@ def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str # --- inputs other tasks produced: fetched from the receiver after the start (Requirement 3.1, 4.4) ------------------- -@pytest.fixture -def plane_receiver(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[egress.Receiver]: - """The reconciler's own receiver on a free port, with a producer's artifact delivered and the task ``stub`` running.""" - monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") - server = egress.Receiver("r", tmp_path / "run", 0) - (tmp_path / "run" / "repo-facts").mkdir(parents=True) - (tmp_path / "run" / "repo-facts" / "facts.json").write_bytes(b'{"a.py": ["f"]}' * 5000) - server.running.add("stub") - threading.Thread(target=server.serve_forever, daemon=True).start() - try: - yield server - finally: - server.shutdown() - server.server_close() - - -def input_env(server: egress.Receiver, tmp_path: Path) -> dict[str, str]: - return { - "OUSAST_ARTIFACT_URL": server.url, - "OUSAST_RUN": "r", - "OUSAST_TASK": "stub", - "OUSAST_INPUTS": json.dumps({"FACTS": "repo-facts/facts.json"}), - "OUSAST_INPUT_FACTS": str(tmp_path / "ws" / "in" / "repo-facts" / "facts.json"), - } - - -def test_inputs_are_fetched_from_the_receiver_before_the_command( - env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - plane_receiver.inputs["stub"] = frozenset({"repo-facts/facts.json"}) - code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path)) - assert code == 0 - fetched = tmp_path / "ws" / "in" / "repo-facts" / "facts.json" - assert fetched.read_bytes() == (tmp_path / "run" / "repo-facts" / "facts.json").read_bytes() - assert "in/repo-facts/facts.json" in json.loads((output / "facts.json").read_text())["workspace_files"], "there before the command ran" - assert "stub" in plane_receiver.delivered - - -def test_an_input_that_cannot_be_fetched_delivers_a_failed_summary_and_never_runs_the_command( - env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - plane_receiver.inputs["stub"] = frozenset({"repo-facts/other.json"}) # facts.json is not among its inputs: 403 - code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path)) - assert code == EXIT_BY_STATUS["failed"] - assert not (output / "facts.json").exists(), "the command never ran with an input missing" - delivered = json.loads((tmp_path / "run" / "stub" / "summary.json").read_text()) - assert delivered["status"] == "failed" and "input FACTS (repo-facts/facts.json) not fetched: HTTP 403" in delivered["reason"] - assert not (tmp_path / "ws" / "in" / "repo-facts").exists() or not any((tmp_path / "ws" / "in" / "repo-facts").iterdir()) - - -def test_fetch_dials_the_override_and_sends_the_url_host(tmp_path: Path) -> None: - seen: list[dict[str, str]] = [] - - class _Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: # noqa: N802 - seen.append({"path": self.path, **dict(self.headers.items())}) - self.send_response(200) - self.send_header("Content-Length", "2") - self.end_headers() - self.wfile.write(b"ok") - - def log_message(self, format: str, *args: object) -> None: # noqa: A002 - pass - - server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - try: - dial = f"127.0.0.1:{server.server_address[1]}" - size = runner.fetch("http://receiver.example/inputs/p/a.json", {"X-Ousast-Run": "r"}, tmp_path / "x" / "a.json", dial=dial) - finally: - server.shutdown() - server.server_close() - assert size == 2 and (tmp_path / "x" / "a.json").read_bytes() == b"ok" - assert seen[0]["path"] == "/inputs/p/a.json" and seen[0]["Host"] == "receiver.example" and seen[0]["X-Ousast-Run"] == "r" - - # --- store delivery through the start body (plane-on-kubernetes 2.2, design section 3) --------------------------- @@ -905,7 +717,7 @@ def test_output_is_put_to_the_presigned_url_and_inputs_are_fetched_from_theirs( env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: """End to end against the S3-shaped fake: the start carries the URLs, the runner PUTs the tar and GETs the input; - no receiver, no OUSAST_ARTIFACT_URL, no URL in any log line.""" + no URL in any log line.""" caplog.set_level(logging.DEBUG) object_store._put("runs/run-1/repo-facts/facts.json", b'{"a.py": ["f"]}') put_url = object_store.presign_put("runs/run-1/stub/output.tar") @@ -964,9 +776,9 @@ def test_a_failed_put_is_retried_alone_on_the_next_start_with_a_fresh_url( def test_deliver_puts_and_names_only_the_safe_url_on_failure(object_store: HttpObjectStore) -> None: url = object_store.presign_put("k/output.tar") - assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, method="PUT") == 200 + assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0) == 200 assert object_store.objects["k/output.tar"] == b"tar" with pytest.raises(runner.DeliveryError) as caught: - runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0, method="PUT") + runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0) assert "HTTP 404" in str(caught.value) and "X-Amz" not in str(caught.value) and "/other/k/output.tar" in str(caught.value) assert runner.safe_url("https://h:1/p/q?X-Amz-Signature=s") == "https://h:1/p/q" diff --git a/tests/test_push_inputs.py b/tests/test_push_inputs.py index c28da8a..707ff6d 100644 --- a/tests/test_push_inputs.py +++ b/tests/test_push_inputs.py @@ -6,7 +6,7 @@ import subprocess import pytest -from openultrasast.push_inputs import validate_manifest +from benchmarks.push.push_inputs import validate_manifest def git(root, *args): @@ -118,7 +118,7 @@ def test_authored_edit_requires_exact_original_and_provenance(manifest): @pytest.mark.parametrize("field", ["snapshots", "cases"]) def test_empty_population_rejected(manifest, field): - from openultrasast.push_inputs import InputError + from benchmarks.push.push_inputs import InputError data, _, run = manifest data[field] = [] @@ -127,7 +127,7 @@ def test_empty_population_rejected(manifest, field): def test_missing_snapshot_identity_rejected(manifest): - from openultrasast.push_inputs import InputError + from benchmarks.push.push_inputs import InputError data, _, run = manifest del data["snapshots"][0]["id"] diff --git a/tests/test_rank.py b/tests/test_rank.py index 5c03657..9fac084 100644 --- a/tests/test_rank.py +++ b/tests/test_rank.py @@ -5,14 +5,7 @@ import pytest from openultrasast.preprocess import FileTarget from openultrasast.provider.openrouter import OpenRouterError, parse_json_content -from openultrasast.rank import ( - RankingError, - composite_priority, - parse_ranker_response, - rank_targets, - rank_targets_with_model, - write_rankings, -) +from openultrasast.rank import RankingError, composite_priority, parse_ranker_response, rank_targets, write_rankings class FakeRankerClient: @@ -72,19 +65,6 @@ def test_parse_ranker_response_rejects_malformed_scores() -> None: parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model") -def test_rank_targets_with_model_chunks_requests() -> None: - targets = [ - FileTarget(path="a.py", absolute_path="/repo/a.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False), - FileTarget(path="b.py", absolute_path="/repo/b.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False), - ] - client = FakeRankerClient() - - rankings = rank_targets_with_model(targets, client=client, model="openrouter/test-ranker", chunk_size=1) - - assert client.calls == 2 - assert {ranking.model_id for ranking in rankings} == {"openrouter/test-ranker"} - - def test_parse_json_content_accepts_fenced_json() -> None: assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []} diff --git a/tests/test_search_coordinator.py b/tests/test_search_coordinator.py index aaf9033..ed5c37e 100644 --- a/tests/test_search_coordinator.py +++ b/tests/test_search_coordinator.py @@ -169,33 +169,6 @@ def test_claim_is_visible_in_worker_snapshot(tmp_path): Coordinator(board(tmp_path), execute, budget=SearchBudget(reason_rounds=1)).run() -def test_batch_adapter_offline_checkpoint_and_step_dispatch(tmp_path, monkeypatch): - from dataclasses import asdict - from pathlib import Path - - monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1])) - from openultrasast.search.coordinator import BatchExecutor, SearchTask - - calls = [] - - def lane(item, output, attempt): - import json - - spec = json.loads(item.inputs["SPEC_URL"]) - calls.append(spec) - assert item.extra_env["SEARCH_STEP"] == spec["step"] - return {"status": "ok", "cost_usd": 0, "intents": []} - - adapter = BatchExecutor( - image="registry/task@sha256:" + "a" * 64, command=("python3", "/worker.py"), output=tmp_path, lanes={"ax": lane} - ) - for index, step in enumerate(("reason", "explore", "verify")): - task = SearchTask(f"task-{index}", "search-1", step, "facts: []", {}, asdict(SearchBudget())) - assert adapter(task)["status"] == "ok" - assert adapter(task)["status"] == "ok" - assert len(calls) == 3 - - def test_invalid_identifiers_refused_before_paths(tmp_path): from dataclasses import asdict diff --git a/tests/test_verification.py b/tests/test_verification.py index 3758c6d..422740d 100644 --- a/tests/test_verification.py +++ b/tests/test_verification.py @@ -10,22 +10,11 @@ from openultrasast.verification import ( VerificationStatus, build_independent_verifier_context, is_report_verified, - validate_evidence_transition, verify_finding, write_verification_results, ) -def test_evidence_transition_rejects_backward_and_skipped_levels() -> None: - assert validate_evidence_transition("suspicion", "static_corroboration") == EvidenceLevel.STATIC_CORROBORATION - - with pytest.raises(ValueError, match="backward"): - validate_evidence_transition("root_cause_explained", "static_corroboration") - - with pytest.raises(ValueError, match="skip"): - validate_evidence_transition("suspicion", "root_cause_explained") - - def test_verified_status_requires_static_corroboration_and_acceptance() -> None: assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED) assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED)