mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f5a904b7f2
commit
f63703a604
+8
-7
@@ -8,6 +8,7 @@
|
|||||||
# parked = no work lands; scaffolding (stubs, unused flags, dead task modules) is removed; spec stays as history
|
# parked = no work lands; scaffolding (stubs, unused flags, dead task modules) is removed; spec stays as history
|
||||||
# retiring = the "closes" deletion is scheduled; the spec directory goes with it
|
# retiring = the "closes" deletion is scheduled; the spec directory goes with it
|
||||||
# review = Pathfinder 2026-10-13 decides between parked and retiring
|
# review = Pathfinder 2026-10-13 decides between parked and retiring
|
||||||
|
# removed = the "closes" deletion has run; the entry stays one release as the record of what went
|
||||||
#
|
#
|
||||||
# open_fronts (the weekly number) = specs with status active or review
|
# open_fronts (the weekly number) = specs with status active or review
|
||||||
# + capabilities with status review or retiring whose "closes" has not run.
|
# + capabilities with status review or retiring whose "closes" has not run.
|
||||||
@@ -181,7 +182,7 @@ status = "active"
|
|||||||
spec = "hunter-learning-loop"
|
spec = "hunter-learning-loop"
|
||||||
modules = ["tool_hunter.py", "hunter_tools.py", "provider/openrouter.py", "model/endpoint.py", "plane/budget.py"]
|
modules = ["tool_hunter.py", "hunter_tools.py", "provider/openrouter.py", "model/endpoint.py", "plane/budget.py"]
|
||||||
lines = 1700
|
lines = 1700
|
||||||
note = "Pathfinder U6 counts the chat-client classes; expected outcome: one client protocol, one metering path"
|
note = "four never-enabled hunter tools removed 2026-10-13 (P0); P2 collapses the client stack to one client, one meter, one resolver"
|
||||||
|
|
||||||
[capability.learn]
|
[capability.learn]
|
||||||
status = "active"
|
status = "active"
|
||||||
@@ -198,7 +199,7 @@ lines = 4803
|
|||||||
[capability.core]
|
[capability.core]
|
||||||
status = "active"
|
status = "active"
|
||||||
modules = ["cli.py", "config.py", "contracts.py", "preprocess.py", "findings.py", "rank.py", "mapping.py", "ruleset", "policy", "reports.py", "stages.py", "run.py", "provenance.py", "profiling.py"]
|
modules = ["cli.py", "config.py", "contracts.py", "preprocess.py", "findings.py", "rank.py", "mapping.py", "ruleset", "policy", "reports.py", "stages.py", "run.py", "provenance.py", "profiling.py"]
|
||||||
note = "hunter.py is the regex quick-rule pool and should be renamed; harness.py processor machinery is unreachable (U1)"
|
note = "hunter.py is the regex quick-rule pool and should be renamed"
|
||||||
|
|
||||||
[capability.pairs-and-benchmarks]
|
[capability.pairs-and-benchmarks]
|
||||||
status = "active"
|
status = "active"
|
||||||
@@ -214,13 +215,13 @@ lines = 4885
|
|||||||
closes = "Pathfinder U8 lists which modules only the retired HarnessX loop used (engine_alerts, harvest, workspaces, profile are the suspects); delete those"
|
closes = "Pathfinder U8 lists which modules only the retired HarnessX loop used (engine_alerts, harvest, workspaces, profile are the suspects); delete those"
|
||||||
|
|
||||||
[capability.regress]
|
[capability.regress]
|
||||||
status = "review"
|
status = "removed" # f5a904b, 2026-10-13: deep mode records regress_retired; proof is the search line
|
||||||
modules = ["regress"]
|
modules = ["regress"]
|
||||||
lines = 549
|
lines = 549
|
||||||
closes = "Pathfinder U9: if search/ covers the triggerable verdict, delete regress/ and the deep-mode REGRESS branch in cli.py:839-880"
|
closes = "Pathfinder U9: if search/ covers the triggerable verdict, delete regress/ and the deep-mode REGRESS branch in cli.py:839-880"
|
||||||
|
|
||||||
[capability.improve]
|
[capability.improve]
|
||||||
status = "retiring"
|
status = "removed" # 94e96c6, 2026-10-13
|
||||||
modules = ["improve"]
|
modules = ["improve"]
|
||||||
lines = 1175
|
lines = 1175
|
||||||
note = "HarnessX-era improve lever; no active spec names it"
|
note = "HarnessX-era improve lever; no active spec names it"
|
||||||
@@ -228,13 +229,13 @@ closes = "delete improve/, `ousast improve` (cli.py:237, 433, 1106), plane/tasks
|
|||||||
|
|
||||||
[capability.scan-extras]
|
[capability.scan-extras]
|
||||||
status = "review"
|
status = "review"
|
||||||
modules = ["fusion.py", "verification.py", "calibration.py", "scoring", "harness.py (processors)"]
|
modules = ["fusion.py", "verification.py", "calibration.py", "scoring"] # harness.py processor machinery removed 2026-10-13
|
||||||
closes = "U1 found fusion decisions never reach findings or score, verification runs before MAP, processors unreachable; decide keep-and-fix or delete per module"
|
closes = "U1 found fusion decisions never reach findings or score, verification runs before MAP, processors unreachable; decide keep-and-fix or delete per module"
|
||||||
|
|
||||||
[capability.mcp-and-skills]
|
[capability.mcp-and-skills]
|
||||||
status = "parked"
|
status = "parked"
|
||||||
spec = "mcp-shift-left-agent"
|
spec = "mcp-shift-left-agent"
|
||||||
modules = ["mcp.py", "skills.py", "index.py"]
|
modules = ["mcp.py"] # skills.py and index.py removed f5a904b, 2026-10-13
|
||||||
lines = 830
|
lines = 830
|
||||||
closes = "see spec.mcp-shift-left-agent"
|
closes = "see spec.mcp-shift-left-agent"
|
||||||
|
|
||||||
@@ -246,6 +247,6 @@ closes = "U10: keep gate.py (CI + plane loop); delete map_gate.py and pair_gate.
|
|||||||
|
|
||||||
[capability.push-evaluation-instruments]
|
[capability.push-evaluation-instruments]
|
||||||
status = "review"
|
status = "review"
|
||||||
modules = ["push_inputs.py", "push_scoring.py"]
|
modules = ["push_scoring.py"] # push_inputs.py moved to benchmarks/push 2026-10-13; push_scoring stays (eligibility.py:18)
|
||||||
lines = 462
|
lines = 462
|
||||||
closes = "U3: move under benchmarks/push or delete if only tests use them"
|
closes = "U3: move under benchmarks/push or delete if only tests use them"
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Reproducible push evaluation inputs
|
# Reproducible push evaluation inputs
|
||||||
|
|
||||||
`inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run.
|
`inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run.
|
||||||
Run `.venv/bin/python -m openultrasast.push_inputs benchmarks/push/inputs.json`.
|
Run `.venv/bin/python -m benchmarks.push.push_inputs benchmarks/push/inputs.json`.
|
||||||
It is offline by default, even if the ordinary repository network environment variable is set.
|
It is offline by default, even if the ordinary repository network environment variable is set.
|
||||||
`--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits.
|
`--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits.
|
||||||
No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one
|
No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one
|
||||||
|
|||||||
@@ -26,11 +26,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|||||||
|
|
||||||
from measure import git, pin_snapshot # noqa: E402
|
from measure import git, pin_snapshot # noqa: E402
|
||||||
|
|
||||||
|
from benchmarks.push.push_inputs import validate_manifest # noqa: E402
|
||||||
from openultrasast.model.scan import ScanBudget # noqa: E402
|
from openultrasast.model.scan import ScanBudget # noqa: E402
|
||||||
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
|
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
|
||||||
from openultrasast.push.runner import _provenance # noqa: E402
|
from openultrasast.push.runner import _provenance # noqa: E402
|
||||||
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
|
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
|
||||||
from openultrasast.push_inputs import validate_manifest # noqa: E402
|
|
||||||
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
|
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
|
||||||
|
|
||||||
TARGET = {"path": "app/routes/contributions.js", "function": "handleContributionsUpdate", "family": "injection"}
|
TARGET = {"path": "app/routes/contributions.js", "function": "handleContributionsUpdate", "family": "injection"}
|
||||||
|
|||||||
@@ -13,10 +13,10 @@ import time
|
|||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from benchmarks.push.push_inputs import validate_manifest
|
||||||
from openultrasast.model.contracts import QuestionIdentity
|
from openultrasast.model.contracts import QuestionIdentity
|
||||||
from openultrasast.model.scan import ScanBudget
|
from openultrasast.model.scan import ScanBudget
|
||||||
from openultrasast.push.runner import _provenance
|
from openultrasast.push.runner import _provenance
|
||||||
from openultrasast.push_inputs import validate_manifest
|
|
||||||
from openultrasast.push_scoring import freeze_profile, score
|
from openultrasast.push_scoring import freeze_profile, score
|
||||||
from openultrasast.repos import checkout_path, load_repo_recipe
|
from openultrasast.repos import checkout_path, load_repo_recipe
|
||||||
|
|
||||||
@@ -314,7 +314,7 @@ def main() -> int:
|
|||||||
"reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.",
|
"reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.",
|
||||||
"selection": args.selection,
|
"selection": args.selection,
|
||||||
"analysis_deadline_seconds": args.deadline,
|
"analysis_deadline_seconds": args.deadline,
|
||||||
"recorded_cases": len(run["records"]),
|
"case_count": len(run["records"]),
|
||||||
"declared_population": len(manifest["cases"]),
|
"declared_population": len(manifest["cases"]),
|
||||||
"core_unchanged": True,
|
"core_unchanged": True,
|
||||||
"adapter_changes": [],
|
"adapter_changes": [],
|
||||||
|
|||||||
@@ -31,11 +31,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|||||||
|
|
||||||
from measure import git, pin_snapshot # noqa: E402
|
from measure import git, pin_snapshot # noqa: E402
|
||||||
|
|
||||||
|
from benchmarks.push.push_inputs import validate_manifest # noqa: E402
|
||||||
from openultrasast.model.scan import ScanBudget # noqa: E402
|
from openultrasast.model.scan import ScanBudget # noqa: E402
|
||||||
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
|
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
|
||||||
from openultrasast.push.runner import _provenance # noqa: E402
|
from openultrasast.push.runner import _provenance # noqa: E402
|
||||||
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
|
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
|
||||||
from openultrasast.push_inputs import validate_manifest # noqa: E402
|
|
||||||
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
|
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
|
||||||
|
|
||||||
# What each declared comparison is expected to show, stated before anything runs. `introduce`
|
# What each declared comparison is expected to show, stated before anything runs. `introduce`
|
||||||
|
|||||||
@@ -26,11 +26,7 @@ flowchart TD
|
|||||||
op -- "otherwise" --> adv["ADVISORY"]
|
op -- "otherwise" --> adv["ADVISORY"]
|
||||||
```
|
```
|
||||||
|
|
||||||
These decisions are meant for changed code. `learn/features.py` builds feature records in two ways:
|
The plane task uses `learn/features.py` to build candidate feature records from delivered artifacts.
|
||||||
|
|
||||||
- for a whole-repository scan;
|
|
||||||
- for a base..head delta (`build_for_scan` with `Delta`). This is the same delta `ousast pre-push` checks.
|
|
||||||
|
|
||||||
Integration into scan and pre-push is still ahead.
|
Integration into scan and pre-push is still ahead.
|
||||||
|
|
||||||
## 1. Instruments produce signals, not verdicts
|
## 1. Instruments produce signals, not verdicts
|
||||||
|
|||||||
@@ -8,10 +8,6 @@ REPO="$(cd "$(dirname "$0")/../.." && pwd)"
|
|||||||
PY="${OUSAST_PYTHON:-$REPO/.venv/bin/python}"
|
PY="${OUSAST_PYTHON:-$REPO/.venv/bin/python}"
|
||||||
RUNNER_IMAGE="$(PYTHONPATH="$REPO/src" "$PY" -m openultrasast.plane.profile --profile "${OUSAST_PLANE_PROFILE:-kind}" --print images.runner)"
|
RUNNER_IMAGE="$(PYTHONPATH="$REPO/src" "$PY" -m openultrasast.plane.profile --profile "${OUSAST_PLANE_PROFILE:-kind}" --print images.runner)"
|
||||||
export RUNNER_IMAGE
|
export RUNNER_IMAGE
|
||||||
# Actors reach the receiver by its Service name through the egress gateway (ops/ax/receiver-service.yaml.tmpl,
|
|
||||||
# applied by up.sh, endpoint this host at OUSAST_ARTIFACT_PORT): the runner dials the Service's ClusterIP on port
|
|
||||||
# 80 with that name as Host; the reconciler looks the ClusterIP up unless OUSAST_ARTIFACT_DIAL is set.
|
|
||||||
export OUSAST_ARTIFACT_PORT="${OUSAST_ARTIFACT_PORT:-18090}"
|
|
||||||
envsubst < "$(dirname "$0")/smoke-run.yaml.tmpl" > "$SRC/smoke-run.yaml"
|
envsubst < "$(dirname "$0")/smoke-run.yaml.tmpl" > "$SRC/smoke-run.yaml"
|
||||||
ousast() { PYTHONPATH="$REPO/src" "$PY" -m openultrasast.cli "$@"; } # this checkout's code, whatever python runs it
|
ousast() { PYTHONPATH="$REPO/src" "$PY" -m openultrasast.cli "$@"; } # this checkout's code, whatever python runs it
|
||||||
ousast plane run "$SRC/smoke-run.yaml" "$@"
|
ousast plane run "$SRC/smoke-run.yaml" "$@"
|
||||||
|
|||||||
@@ -47,16 +47,6 @@ class FalsePositiveLearning:
|
|||||||
demotion: float = 0.5
|
demotion: float = 0.5
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class RankingMetrics:
|
|
||||||
tier: str
|
|
||||||
verified_findings: int
|
|
||||||
false_positives: int
|
|
||||||
time_to_verification_seconds: float
|
|
||||||
missed_fixture_vulnerabilities: int
|
|
||||||
false_positive_reasons: dict[str, int]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class RankingCalibration:
|
class RankingCalibration:
|
||||||
path: str
|
path: str
|
||||||
@@ -105,38 +95,6 @@ def learning_from_finding_outcome(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def compute_ranking_metrics(events: list[dict[str, object]], *, tier: str) -> RankingMetrics:
|
|
||||||
verified = 0
|
|
||||||
false_positives = 0
|
|
||||||
reason_counts: dict[str, int] = {}
|
|
||||||
verification_times: list[float] = []
|
|
||||||
missed = 0
|
|
||||||
for event in events:
|
|
||||||
if event.get("tier") != tier:
|
|
||||||
continue
|
|
||||||
outcome = event.get("outcome")
|
|
||||||
if outcome == FindingOutcome.VERIFIED:
|
|
||||||
verified += 1
|
|
||||||
if outcome in _FALSE_POSITIVE_OUTCOMES:
|
|
||||||
false_positives += 1
|
|
||||||
reason = str(event.get("reason", FalsePositiveReason.UNVERIFIED))
|
|
||||||
reason_counts[reason] = reason_counts.get(reason, 0) + 1
|
|
||||||
duration = event.get("time_to_verification_seconds")
|
|
||||||
if isinstance(duration, int | float) and not isinstance(duration, bool):
|
|
||||||
verification_times.append(float(duration))
|
|
||||||
missed_value = event.get("missed_fixture_vulnerabilities", 0)
|
|
||||||
if isinstance(missed_value, int) and not isinstance(missed_value, bool):
|
|
||||||
missed += missed_value
|
|
||||||
return RankingMetrics(
|
|
||||||
tier=tier,
|
|
||||||
verified_findings=verified,
|
|
||||||
false_positives=false_positives,
|
|
||||||
time_to_verification_seconds=round(sum(verification_times) / len(verification_times), 2) if verification_times else 0.0,
|
|
||||||
missed_fixture_vulnerabilities=missed,
|
|
||||||
false_positive_reasons=dict(sorted(reason_counts.items())),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearning]) -> RankingCalibration:
|
def calibrate_ranking(ranking: RankingScore, learnings: list[FalsePositiveLearning]) -> RankingCalibration:
|
||||||
applicable = [learning for learning in learnings if _learning_applies(ranking.path, learning)]
|
applicable = [learning for learning in learnings if _learning_applies(ranking.path, learning)]
|
||||||
calibrated = ranking.priority
|
calibrated = ranking.priority
|
||||||
|
|||||||
@@ -548,7 +548,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
|||||||
config=config,
|
config=config,
|
||||||
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets),
|
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets),
|
||||||
)
|
)
|
||||||
write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json")
|
write_harness_config(config=config, path=run.root / "harness.json")
|
||||||
runtime.start(mode=mode, target=run.target)
|
runtime.start(mode=mode, target=run.target)
|
||||||
policy = runtime.run_stage("policy_load", load_policy)
|
policy = runtime.run_stage("policy_load", load_policy)
|
||||||
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
||||||
|
|||||||
@@ -99,17 +99,6 @@ def uncovered_languages(
|
|||||||
return sorted(counts.items())
|
return sorted(counts.items())
|
||||||
|
|
||||||
|
|
||||||
def build_quick_hunter_prompt(target: FileTarget, source_excerpt: str) -> str:
|
|
||||||
return (
|
|
||||||
"Review this file for security-relevant issues. Return structured findings only when evidence exists.\n"
|
|
||||||
f"Path: {target.path}\n"
|
|
||||||
f"Language: {target.language}\n"
|
|
||||||
f"Tags: {', '.join(target.tags) or 'none'}\n"
|
|
||||||
"Source excerpt:\n"
|
|
||||||
f"{source_excerpt[:4000]}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_findings(findings: list[StaticFinding], path: Path) -> None:
|
def write_findings(findings: list[StaticFinding], path: Path) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
payload = {"findings": [asdict(finding) for finding in findings]}
|
payload = {"findings": [asdict(finding) for finding in findings]}
|
||||||
|
|||||||
@@ -91,10 +91,6 @@ def fusion_triggers(
|
|||||||
return sorted(triggers)
|
return sorted(triggers)
|
||||||
|
|
||||||
|
|
||||||
def should_fuse(finding: StaticFinding, verification: VerificationResult | None = None, *, high_assurance: bool = False) -> bool:
|
|
||||||
return bool(fusion_triggers(finding, verification, high_assurance=high_assurance))
|
|
||||||
|
|
||||||
|
|
||||||
def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict:
|
def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict:
|
||||||
"""Vulnerability-leaning panel: steel-mans the vulnerability case."""
|
"""Vulnerability-leaning panel: steel-mans the vulnerability case."""
|
||||||
reachable = finding.reachability_status == "reachable"
|
reachable = finding.reachability_status == "reachable"
|
||||||
@@ -213,5 +209,4 @@ __all__ = [
|
|||||||
"fuse_finding",
|
"fuse_finding",
|
||||||
"fuse_findings",
|
"fuse_findings",
|
||||||
"fusion_triggers",
|
"fusion_triggers",
|
||||||
"should_fuse",
|
]
|
||||||
]
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from collections.abc import Callable
|
|||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass, field
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Literal, Protocol
|
from typing import Any, Literal
|
||||||
|
|
||||||
from .config import ResolvedConfig
|
from .config import ResolvedConfig
|
||||||
|
|
||||||
@@ -21,11 +21,6 @@ EventType = Literal[
|
|||||||
"task_end",
|
"task_end",
|
||||||
]
|
]
|
||||||
|
|
||||||
ContractMode = Literal["strict", "warn"]
|
|
||||||
|
|
||||||
|
|
||||||
class HarnessContractError(ValueError):
|
|
||||||
"""Raised when a processor violates its declared state contract."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -37,27 +32,9 @@ class HarnessEvent:
|
|||||||
payload: dict[str, Any] = field(default_factory=dict)
|
payload: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ProcessorSpec:
|
|
||||||
name: str
|
|
||||||
version: str
|
|
||||||
handles: tuple[EventType, ...]
|
|
||||||
reads: tuple[str, ...] = ()
|
|
||||||
writes: tuple[str, ...] = ()
|
|
||||||
|
|
||||||
|
|
||||||
class Processor(Protocol):
|
|
||||||
spec: ProcessorSpec
|
|
||||||
|
|
||||||
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any] | None:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class SerializedHarnessConfig:
|
class SerializedHarnessConfig:
|
||||||
contract_mode: ContractMode
|
|
||||||
model_roles: dict[str, str | None]
|
model_roles: dict[str, str | None]
|
||||||
processor_versions: dict[str, str]
|
|
||||||
prompt_hashes: dict[str, str]
|
prompt_hashes: dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
@@ -84,14 +61,10 @@ class HarnessRuntime:
|
|||||||
scan_id: str,
|
scan_id: str,
|
||||||
config: ResolvedConfig,
|
config: ResolvedConfig,
|
||||||
trace_writer: HarnessTraceWriter,
|
trace_writer: HarnessTraceWriter,
|
||||||
processors: list[Processor] | None = None,
|
|
||||||
contract_mode: ContractMode = "strict",
|
|
||||||
) -> None:
|
) -> None:
|
||||||
self.scan_id = scan_id
|
self.scan_id = scan_id
|
||||||
self.config = config
|
self.config = config
|
||||||
self.trace_writer = trace_writer
|
self.trace_writer = trace_writer
|
||||||
self.processors = processors or []
|
|
||||||
self.contract_mode = contract_mode
|
|
||||||
self.state: dict[str, Any] = {"degradations": []}
|
self.state: dict[str, Any] = {"degradations": []}
|
||||||
|
|
||||||
def start(self, *, mode: str, target: Path) -> None:
|
def start(self, *, mode: str, target: Path) -> None:
|
||||||
@@ -119,71 +92,22 @@ class HarnessRuntime:
|
|||||||
payload=payload,
|
payload=payload,
|
||||||
)
|
)
|
||||||
self.trace_writer.append(event)
|
self.trace_writer.append(event)
|
||||||
for processor in self.processors:
|
|
||||||
if event_type in processor.spec.handles:
|
|
||||||
self._run_processor(processor, event)
|
|
||||||
|
|
||||||
def _run_processor(self, processor: Processor, event: HarnessEvent) -> None:
|
|
||||||
before = set(self.state)
|
|
||||||
missing_reads = [key for key in processor.spec.reads if key not in self.state]
|
|
||||||
if missing_reads:
|
|
||||||
self._contract_violation(processor.spec.name, f"missing reads: {', '.join(missing_reads)}")
|
|
||||||
return
|
|
||||||
|
|
||||||
updates = processor.handle(event, dict(self.state)) or {}
|
|
||||||
if not isinstance(updates, dict):
|
|
||||||
self._contract_violation(processor.spec.name, "processor returned a non-dict update")
|
|
||||||
return
|
|
||||||
undeclared = sorted(set(updates) - set(processor.spec.writes))
|
|
||||||
if undeclared:
|
|
||||||
self._contract_violation(processor.spec.name, f"undeclared writes: {', '.join(undeclared)}")
|
|
||||||
return
|
|
||||||
self.state.update(updates)
|
|
||||||
changed = sorted(set(self.state) - before | set(updates))
|
|
||||||
self.trace_writer.append(
|
|
||||||
HarnessEvent(
|
|
||||||
event_type=event.event_type,
|
|
||||||
scan_id=self.scan_id,
|
|
||||||
stage=event.stage,
|
|
||||||
timestamp=datetime.now(UTC).isoformat(),
|
|
||||||
payload={"processor": processor.spec.name, "changed_state": changed},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def _contract_violation(self, processor_name: str, reason: str) -> None:
|
|
||||||
message = f"processor {processor_name} contract violation: {reason}"
|
|
||||||
if self.contract_mode == "strict":
|
|
||||||
raise HarnessContractError(message)
|
|
||||||
self.state.setdefault("degradations", []).append(message)
|
|
||||||
self.trace_writer.append(
|
|
||||||
HarnessEvent(
|
|
||||||
event_type="stage_end",
|
|
||||||
scan_id=self.scan_id,
|
|
||||||
stage="processor_contract",
|
|
||||||
timestamp=datetime.now(UTC).isoformat(),
|
|
||||||
payload={"status": "degraded", "reason": message},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_harness_config(
|
def write_harness_config(
|
||||||
*,
|
*,
|
||||||
config: ResolvedConfig,
|
config: ResolvedConfig,
|
||||||
processors: list[Processor],
|
|
||||||
contract_mode: ContractMode,
|
|
||||||
path: Path,
|
path: Path,
|
||||||
prompts: dict[str, str] | None = None,
|
prompts: dict[str, str] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
prompts = prompts or {}
|
prompts = prompts or {}
|
||||||
serialized = SerializedHarnessConfig(
|
serialized = SerializedHarnessConfig(
|
||||||
contract_mode=contract_mode,
|
|
||||||
model_roles={
|
model_roles={
|
||||||
"ranker": config.models.ranker,
|
"ranker": config.models.ranker,
|
||||||
"hunter": config.models.hunter,
|
"hunter": config.models.hunter,
|
||||||
"verifier": None, # schema kept byte for byte; LLM verification runs on the plane
|
"verifier": None, # schema kept byte for byte; LLM verification runs on the plane
|
||||||
"patcher": None, # schema kept; `[models] patcher` was retired 2026-10-02 (never read)
|
"patcher": None, # schema kept; `[models] patcher` was retired 2026-10-02 (never read)
|
||||||
},
|
},
|
||||||
processor_versions={processor.spec.name: processor.spec.version for processor in processors},
|
|
||||||
prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())},
|
prompt_hashes={name: _hash_text(prompt) for name, prompt in sorted(prompts.items())},
|
||||||
)
|
)
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from collections.abc import Iterable, Iterator, Mapping
|
from collections.abc import Iterator
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .preprocess import FileTarget, enumerate_source_files
|
from .preprocess import enumerate_source_files
|
||||||
|
|
||||||
|
|
||||||
class PathEscapesRepo(ValueError):
|
class PathEscapesRepo(ValueError):
|
||||||
@@ -80,25 +80,9 @@ def grep_repo(root: Path, pattern: str, *, max_matches: int) -> list[dict[str, o
|
|||||||
return matches
|
return matches
|
||||||
|
|
||||||
|
|
||||||
def find_refs(root: Path, symbol: str, mapping_index: object) -> list[dict[str, object]]:
|
def find_refs(root: Path, symbol: str) -> list[dict[str, object]]:
|
||||||
resolved_root = root.resolve()
|
resolved_root = root.resolve()
|
||||||
refs: list[dict[str, object]] = []
|
refs: list[dict[str, object]] = []
|
||||||
for record in _iter_mapping_records(mapping_index):
|
|
||||||
if not _record_matches_symbol(record, symbol):
|
|
||||||
continue
|
|
||||||
raw_path = _record_path(record)
|
|
||||||
if raw_path is None:
|
|
||||||
continue
|
|
||||||
clamped = clamp_repo_path(resolved_root, raw_path)
|
|
||||||
name = _record_value(record, "name", "function_name", "symbol")
|
|
||||||
refs.append(
|
|
||||||
{
|
|
||||||
"path": clamped.relative_to(resolved_root).as_posix(),
|
|
||||||
"name": str(name) if name is not None else symbol,
|
|
||||||
"line": _as_line(_record_value(record, "line")),
|
|
||||||
"source": "mapping",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
if symbol:
|
if symbol:
|
||||||
compiled = re.compile(rf"\b{re.escape(symbol)}\b")
|
compiled = re.compile(rf"\b{re.escape(symbol)}\b")
|
||||||
for relative, lines in _source_lines(resolved_root):
|
for relative, lines in _source_lines(resolved_root):
|
||||||
@@ -113,221 +97,3 @@ def _iter_clamped_source_files(root: Path) -> Iterator[tuple[str, Path]]:
|
|||||||
for path in enumerate_source_files(resolved_root):
|
for path in enumerate_source_files(resolved_root):
|
||||||
relative = path.relative_to(resolved_root).as_posix()
|
relative = path.relative_to(resolved_root).as_posix()
|
||||||
yield relative, clamp_repo_path(resolved_root, relative)
|
yield relative, clamp_repo_path(resolved_root, relative)
|
||||||
|
|
||||||
|
|
||||||
def _iter_mapping_records(mapping_index: object) -> Iterable[object]:
|
|
||||||
if mapping_index is None:
|
|
||||||
return ()
|
|
||||||
if isinstance(mapping_index, Mapping):
|
|
||||||
return (mapping_index,)
|
|
||||||
if isinstance(mapping_index, (str, bytes, bytearray)):
|
|
||||||
return ()
|
|
||||||
if isinstance(mapping_index, Iterable):
|
|
||||||
return mapping_index
|
|
||||||
return (mapping_index,)
|
|
||||||
|
|
||||||
|
|
||||||
def _record_matches_symbol(record: object, symbol: str) -> bool:
|
|
||||||
for field in ("name", "function_name", "symbol"):
|
|
||||||
value = _record_value(record, field)
|
|
||||||
if value is not None and str(value) == symbol:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _record_path(record: object) -> str | None:
|
|
||||||
value = record.get("path") if isinstance(record, Mapping) else getattr(record, "path", None)
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
return str(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _record_value(record: object, *names: str) -> object | None:
|
|
||||||
for name in names:
|
|
||||||
raw: object | None
|
|
||||||
if isinstance(record, Mapping):
|
|
||||||
if name not in record:
|
|
||||||
continue
|
|
||||||
raw = record[name]
|
|
||||||
else:
|
|
||||||
raw = getattr(record, name, None)
|
|
||||||
if raw is not None:
|
|
||||||
return raw
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _as_line(value: object) -> int | None:
|
|
||||||
if isinstance(value, bool) or value is None:
|
|
||||||
return None
|
|
||||||
if isinstance(value, int):
|
|
||||||
return value
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
# --- curated structural tools (learning-harness, Req 6.4, 6.5) --------------------------------------
|
|
||||||
# A detector reads code with `read_file` and `read_definition`; the three list tools below answer with
|
|
||||||
# identifiers and line spans only, so they can never become a second way to dump the repository. Every
|
|
||||||
# path a caller supplies is clamped to the scan root first. No caching: a detector calls these a handful
|
|
||||||
# of times per region, and a stale view of a tree that a round may have rewritten is worse than a reparse.
|
|
||||||
|
|
||||||
|
|
||||||
def read_definition(root: Path, symbol: str, *, max_chars: int, path: str | None = None) -> dict[str, object] | None:
|
|
||||||
"""The definition of one named function: where it is, how far it runs, and its text. None when absent."""
|
|
||||||
resolved_root = root.resolve()
|
|
||||||
scope = clamp_repo_path(resolved_root, path) if path is not None else None
|
|
||||||
for relative, target in _iter_clamped_source_files(resolved_root):
|
|
||||||
if scope is not None and target != scope:
|
|
||||||
continue
|
|
||||||
text = target.read_text(errors="ignore")
|
|
||||||
span = _named_span(relative, text, symbol)
|
|
||||||
if span is None:
|
|
||||||
continue
|
|
||||||
lines = text.splitlines()
|
|
||||||
start, end = span
|
|
||||||
while start > 1 and lines[start - 2].lstrip().startswith("@"):
|
|
||||||
start -= 1 # a handler's decorators are part of what the detector must see
|
|
||||||
body = "\n".join(lines[start - 1 : end])
|
|
||||||
return {"path": relative, "name": symbol, "start_line": start, "end_line": end, "text": body[: max(max_chars, 0)]}
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def entry_points(root: Path, *, path: str | None = None) -> list[dict[str, object]]:
|
|
||||||
"""Handlers the mapper names, with their access classification and evidence. No source text."""
|
|
||||||
from .mapping import analyze_entry_points
|
|
||||||
|
|
||||||
resolved_root = root.resolve()
|
|
||||||
relative = _relative_or_none(resolved_root, path)
|
|
||||||
targets = _repo_targets(resolved_root)
|
|
||||||
rows: list[dict[str, object]] = []
|
|
||||||
for record in analyze_entry_points(resolved_root, targets):
|
|
||||||
if relative is not None and record.path != relative:
|
|
||||||
continue
|
|
||||||
rows.append(
|
|
||||||
{
|
|
||||||
"path": record.path,
|
|
||||||
"function": record.function_name,
|
|
||||||
"name": record.name,
|
|
||||||
"kind": record.kind,
|
|
||||||
"line": record.line,
|
|
||||||
"end_line": record.end_line,
|
|
||||||
"access_level": record.access_level,
|
|
||||||
"evidence": list(record.access_evidence),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
def flows(root: Path, *, path: str, function: str) -> list[dict[str, object]]:
|
|
||||||
"""Adjudicated source-to-sink records inside one function: which source, which sink, what the overlay said."""
|
|
||||||
from .findings import quick_scan_findings
|
|
||||||
from .rank import rank_targets
|
|
||||||
from .semantic import adjudicate
|
|
||||||
|
|
||||||
resolved_root = root.resolve()
|
|
||||||
relative = _relative_or_none(resolved_root, path)
|
|
||||||
targets = _repo_targets(resolved_root)
|
|
||||||
span = _span_in(resolved_root, relative, function)
|
|
||||||
if span is None:
|
|
||||||
return []
|
|
||||||
findings = [item for item in quick_scan_findings(resolved_root, targets, rank_targets(targets), None) if item.status != "shadow"]
|
|
||||||
rows: list[dict[str, object]] = []
|
|
||||||
for record in adjudicate(root=resolved_root, targets=targets, findings=findings):
|
|
||||||
if record.path != relative or record.line is None or not span[0] <= record.line <= span[1]:
|
|
||||||
continue
|
|
||||||
rows.append(
|
|
||||||
{
|
|
||||||
"path": record.path,
|
|
||||||
"function": function,
|
|
||||||
"line": record.line,
|
|
||||||
"sources": list(record.sources),
|
|
||||||
"sinks": list(record.sinks),
|
|
||||||
"disposition": record.disposition,
|
|
||||||
"cwe": record.cwe,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
def obligations(root: Path, *, path: str, function: str) -> list[dict[str, object]]:
|
|
||||||
"""What one function owes and has not discharged: the operation, the missing discharger, why it is owed."""
|
|
||||||
from .config import ObligationsConfig
|
|
||||||
from .semantic.facts import FactLoadError, load_facts
|
|
||||||
from .semantic.ir import parse_file
|
|
||||||
from .semantic.obligations import check_obligations, load_obligation_facts
|
|
||||||
from .semantic.obligations.dominance import OrderDominance
|
|
||||||
|
|
||||||
resolved_root = root.resolve()
|
|
||||||
relative = _relative_or_none(resolved_root, path)
|
|
||||||
targets = _repo_targets(resolved_root)
|
|
||||||
try:
|
|
||||||
flow_facts = load_facts()
|
|
||||||
except FactLoadError:
|
|
||||||
return []
|
|
||||||
irs: dict[str, tuple[object, str]] = {}
|
|
||||||
texts: dict[str, str] = {}
|
|
||||||
for target in targets:
|
|
||||||
try:
|
|
||||||
text = (resolved_root / target.path).read_text(errors="ignore")
|
|
||||||
except OSError:
|
|
||||||
continue
|
|
||||||
texts[target.path] = text
|
|
||||||
irs[target.path] = (parse_file(target.path, text, target.language), text)
|
|
||||||
from .mapping import analyze_entry_points
|
|
||||||
|
|
||||||
result = check_obligations(
|
|
||||||
irs=irs, # type: ignore[arg-type]
|
|
||||||
entries=analyze_entry_points(resolved_root, targets),
|
|
||||||
facts=load_obligation_facts(),
|
|
||||||
flow_facts=flow_facts,
|
|
||||||
policy=None,
|
|
||||||
paths=(),
|
|
||||||
dominance=OrderDominance(texts=texts),
|
|
||||||
store_shapes=(),
|
|
||||||
min_siblings=ObligationsConfig().min_siblings,
|
|
||||||
)
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"path": finding.operation.path,
|
|
||||||
"function": finding.operation.function,
|
|
||||||
"line": finding.operation.line,
|
|
||||||
"operation": finding.operation.kind,
|
|
||||||
"resource": finding.operation.resource,
|
|
||||||
"missing": finding.missing,
|
|
||||||
"label": finding.label,
|
|
||||||
"evidence": list(finding.evidence),
|
|
||||||
}
|
|
||||||
for finding in result.findings
|
|
||||||
if (relative is None or finding.operation.path == relative) and finding.operation.function == function
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _repo_targets(root: Path) -> list[FileTarget]:
|
|
||||||
from .preprocess import preprocess_repository
|
|
||||||
|
|
||||||
_, targets = preprocess_repository(root)
|
|
||||||
return list(targets)
|
|
||||||
|
|
||||||
|
|
||||||
def _relative_or_none(root: Path, path: str | None) -> str | None:
|
|
||||||
if path is None:
|
|
||||||
return None
|
|
||||||
return clamp_repo_path(root, path).relative_to(root).as_posix()
|
|
||||||
|
|
||||||
|
|
||||||
def _named_span(relative: str, text: str, symbol: str) -> tuple[int, int] | None:
|
|
||||||
from .preprocess import detect_language
|
|
||||||
from .semantic.functions import named_function_ranges, spans_named
|
|
||||||
|
|
||||||
ranges = named_function_ranges(relative, text, detect_language(Path(relative)))
|
|
||||||
spans = spans_named(ranges or (), symbol)
|
|
||||||
return spans[0] if spans else None
|
|
||||||
|
|
||||||
|
|
||||||
def _span_in(root: Path, relative: str | None, function: str) -> tuple[int, int] | None:
|
|
||||||
if relative is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
text = (root / relative).read_text(errors="ignore")
|
|
||||||
except OSError:
|
|
||||||
return None
|
|
||||||
return _named_span(relative, text, function)
|
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
"""Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1).
|
"""Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1).
|
||||||
|
|
||||||
No model. The same per-instrument builders serve two callers:
|
The plane task ``features`` (:mod:`..plane.tasks.features`) reads a case's delivered artifacts.
|
||||||
|
|
||||||
- the plane task ``features`` (:mod:`..plane.tasks.features`), which reads a case's delivered artifacts;
|
|
||||||
- the host: :func:`build_for_scan` over a checkout's quick findings and an engine result, for ``scan`` and
|
|
||||||
``pre-push`` (a whole-repository scan, or a base..head delta with :class:`Delta`).
|
|
||||||
|
|
||||||
A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument
|
A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument
|
||||||
emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the
|
emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the
|
||||||
@@ -26,8 +22,8 @@ from typing import Any
|
|||||||
from ..model.taxonomy import load_families
|
from ..model.taxonomy import load_families
|
||||||
from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files
|
from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files
|
||||||
from ..preprocess import build_file_target, detect_language
|
from ..preprocess import build_file_target, detect_language
|
||||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
|
from ..ruleset import PatternRule
|
||||||
from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, infer_for_checkout, origin_of
|
from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, origin_of
|
||||||
from .schema import (
|
from .schema import (
|
||||||
BY_NAME,
|
BY_NAME,
|
||||||
LANGUAGES,
|
LANGUAGES,
|
||||||
@@ -62,12 +58,6 @@ NONE = Part("none")
|
|||||||
NOT_APPLICABLE = Part("not_applicable")
|
NOT_APPLICABLE = Part("not_applicable")
|
||||||
|
|
||||||
|
|
||||||
def language_of(path: str) -> str:
|
|
||||||
"""The closed language of a path (its extension), ``other`` outside :data:`.schema.LANGUAGES`."""
|
|
||||||
language = detect_language(Path(path))
|
|
||||||
return language if language in LANGUAGES else "other"
|
|
||||||
|
|
||||||
|
|
||||||
def _clip(name: str, value: Any) -> Any:
|
def _clip(name: str, value: Any) -> Any:
|
||||||
spec = BY_NAME[name]
|
spec = BY_NAME[name]
|
||||||
if value is None or spec.type != "int" or isinstance(value, bool):
|
if value is None or spec.type != "int" or isinstance(value, bool):
|
||||||
@@ -328,18 +318,6 @@ def function_of(lines: Sequence[str] | None, path: str, language: str, function:
|
|||||||
# --- the host path ---------------------------------------------------------------------------------------------------
|
# --- the host path ---------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Delta:
|
|
||||||
"""A base..head delta: the base checkout, the head-side changed line ranges per path (``git diff -U0``), and the
|
|
||||||
base's quick findings and engine result, whose signals decide novelty."""
|
|
||||||
|
|
||||||
root: Path
|
|
||||||
changed: Mapping[str, Sequence[tuple[int, int]]]
|
|
||||||
findings: Sequence[Any] = ()
|
|
||||||
engine_result: Mapping[str, Any] | None = None
|
|
||||||
pin: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class SourceIndex:
|
class SourceIndex:
|
||||||
"""A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level
|
"""A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level
|
||||||
caller graph (the repo-facts patterns) for the entry-point distance."""
|
caller graph (the repo-facts patterns) for the entry-point distance."""
|
||||||
@@ -423,12 +401,6 @@ def _family_of(rule: PatternRule | None) -> str:
|
|||||||
return family.id if family is not None else UNKNOWN_FAMILY
|
return family.id if family is not None else UNKNOWN_FAMILY
|
||||||
|
|
||||||
|
|
||||||
def _engine_state(result: Mapping[str, Any] | None) -> str:
|
|
||||||
if result is None:
|
|
||||||
return "none"
|
|
||||||
return "ran" if int(result.get("questions") or 0) > 0 else "failed"
|
|
||||||
|
|
||||||
|
|
||||||
def _signals(
|
def _signals(
|
||||||
index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule]
|
index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule]
|
||||||
) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]:
|
) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]:
|
||||||
@@ -447,108 +419,6 @@ def _signals(
|
|||||||
return quick, engine
|
return quick, engine
|
||||||
|
|
||||||
|
|
||||||
def _changed_in(index: SourceIndex, changed: Mapping[str, Sequence[tuple[int, int]]]) -> dict[tuple[str, str], int]:
|
|
||||||
"""(path, function) -> changed lines inside it, by the head's enclosing declarations."""
|
|
||||||
counts: dict[tuple[str, str], int] = {}
|
|
||||||
for path, ranges in changed.items():
|
|
||||||
for lo, hi in ranges:
|
|
||||||
for line in range(int(lo), int(hi) + 1):
|
|
||||||
key = (path, index.function_at(path, line))
|
|
||||||
counts[key] = counts.get(key, 0) + 1
|
|
||||||
return counts
|
|
||||||
|
|
||||||
|
|
||||||
def _body_calls(lines: Sequence[str] | None, language: str, function: str) -> set[str]:
|
|
||||||
if not lines:
|
|
||||||
return set()
|
|
||||||
return {m.group(1) for i, line in enumerate(lines) if enclosing(lines, i, language) == function for m in _CALL.finditer(line)}
|
|
||||||
|
|
||||||
|
|
||||||
def build_for_scan(
|
|
||||||
root: Path,
|
|
||||||
findings: Sequence[Any],
|
|
||||||
engine_result: Mapping[str, Any] | None,
|
|
||||||
*,
|
|
||||||
base: Delta | None = None,
|
|
||||||
rules: Sequence[PatternRule] | None = None,
|
|
||||||
quick_languages: Iterable[str] | None = None,
|
|
||||||
engine_languages: Iterable[str] | None = None,
|
|
||||||
roles: RoleSet | None | bool = True,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
"""``static`` records for a checkout: one per (path, function, family) with a quick or engine signal. ``findings``
|
|
||||||
are the quick findings (enabled and shadow alike), ``engine_result`` the engine's record (``findings`` with
|
|
||||||
``site``/``family``/``rung``/``witness``, ``questions``, ``completed``, ``degradations``; ``None`` when the engine
|
|
||||||
did not run, ``questions == 0`` is ``failed``). With ``base`` only functions enclosing a changed line are
|
|
||||||
candidates, and ``delta.*`` compares their signals with the base's. ``roles``: ``True`` infers wrapper roles over
|
|
||||||
the checkout (priors off), a :class:`.roles.RoleSet` is used as given, ``False``/``None`` leaves ``roles`` ``none``."""
|
|
||||||
from ..plane.tasks.alerts import covers
|
|
||||||
from ..plane.tasks.alerts import engine_languages as engine_covered
|
|
||||||
from ..plane.tasks.alerts import quick_languages as quick_covered
|
|
||||||
from ..semantic.facts import FactLoadError, load_facts
|
|
||||||
|
|
||||||
root = Path(root)
|
|
||||||
loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR)
|
|
||||||
by_id = {rule.rule_id: rule for rule in loaded}
|
|
||||||
quick_langs = set(quick_languages if quick_languages is not None else quick_covered())
|
|
||||||
engine_langs = set(engine_languages if engine_languages is not None else engine_covered())
|
|
||||||
index = SourceIndex(root)
|
|
||||||
quick, engine = _signals(index, findings, engine_result, by_id)
|
|
||||||
keys = sorted(set(quick) | set(engine))
|
|
||||||
changed: dict[tuple[str, str], int] = {}
|
|
||||||
base_quick: dict[tuple[str, str, str], list[tuple[str, str]]] = {}
|
|
||||||
base_engine: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {}
|
|
||||||
base_index: SourceIndex | None = None
|
|
||||||
if base is not None:
|
|
||||||
changed = _changed_in(index, base.changed)
|
|
||||||
keys = [k for k in keys if (k[0], k[1]) in changed]
|
|
||||||
base_index = SourceIndex(base.root)
|
|
||||||
base_quick, base_engine = _signals(base_index, base.findings, base.engine_result, by_id)
|
|
||||||
entries, file_entries = entry_names(root, {k[0] for k in keys}) if keys else (set(), set())
|
|
||||||
callers = index.callers() if keys else {}
|
|
||||||
quick_version = ruleset_digest(loaded)
|
|
||||||
engine_state = _engine_state(engine_result)
|
|
||||||
completed, questions = (engine_result or {}).get("completed"), (engine_result or {}).get("questions")
|
|
||||||
completion = int(completed) / int(questions) if isinstance(completed, int) and isinstance(questions, int) and questions else None
|
|
||||||
degraded = bool((engine_result or {}).get("degradations"))
|
|
||||||
vocabularies: dict[str, Vocabulary] = {}
|
|
||||||
inferred = (infer_for_checkout(root) if keys else RoleSet(sources=("inferred_wrapper",))) if roles is True else roles or None
|
|
||||||
try:
|
|
||||||
sanitizer_calls = {language: {c for s in load_facts().for_language(language).sanitizers for c in s.calls} for language in LANGUAGES}
|
|
||||||
except FactLoadError:
|
|
||||||
sanitizer_calls = {}
|
|
||||||
out: list[dict[str, Any]] = []
|
|
||||||
for key in keys:
|
|
||||||
path, function, family = key
|
|
||||||
language = detect_language(Path(path))
|
|
||||||
parts: dict[str, Part] = {}
|
|
||||||
parts["quick"] = quick_part(quick.get(key, []), by_id, quick_version) if covers(quick_langs, language) else NONE
|
|
||||||
if language in engine_langs and engine_state != "none":
|
|
||||||
vocabulary = vocabularies.setdefault(language, Vocabulary.load(language))
|
|
||||||
parts["engine"] = (
|
|
||||||
engine_part(engine.get(key, []), completion=completion, degraded=degraded, vocabulary=vocabulary)
|
|
||||||
if engine_state == "ran" else Part("failed")
|
|
||||||
) # fmt: skip
|
|
||||||
lines = index.lines(path)
|
|
||||||
parts["source"] = source_part(lines, language, function)
|
|
||||||
parts["roles"] = roles_part(function_of(lines, path, language, function), inferred, language)
|
|
||||||
distance = 0 if function == GLOBAL and path in file_entries else entry_distance(function, entries, callers)
|
|
||||||
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
|
|
||||||
parts["facts"] = facts_part(_callers_of(index, callers, path, function), function)
|
|
||||||
if base is None or base_index is None:
|
|
||||||
parts["delta"] = NOT_APPLICABLE
|
|
||||||
else:
|
|
||||||
head_n = len(quick.get(key, [])) + len(engine.get(key, []))
|
|
||||||
base_n = len(base_quick.get(key, [])) + len(base_engine.get(key, []))
|
|
||||||
novelty = "new" if base_n == 0 else "worsened" if head_n > base_n else "unchanged"
|
|
||||||
known = sanitizer_calls.get(language, set())
|
|
||||||
removed = bool((_body_calls(base_index.lines(path), language, function) - _body_calls(lines, language, function)) & known)
|
|
||||||
values = {"delta.novelty": novelty, "delta.changed_lines": changed.get((path, function), 0), "delta.sanitizer_removed": removed}
|
|
||||||
parts["delta"] = Part("ran", values)
|
|
||||||
unit, pin = ("delta", base.pin) if base is not None else ("pin", None)
|
|
||||||
out.append(record(f"{path}::{function}", family, language, "static", parts, unit=unit, base=pin))
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]:
|
def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]:
|
||||||
"""The call sites of ``function`` in other product files, as repo-facts records them (capped at 40)."""
|
"""The call sites of ``function`` in other product files, as repo-facts records them (capped at 40)."""
|
||||||
if function == GLOBAL or function not in callers:
|
if function == GLOBAL or function not in callers:
|
||||||
@@ -567,8 +437,8 @@ def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str,
|
|||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Delta", "Part", "SourceIndex", "Vocabulary", "agree_part",
|
"CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Part", "SourceIndex", "Vocabulary", "agree_part",
|
||||||
"function_of", "model_sinks_part", "roles_part",
|
"function_of", "model_sinks_part", "roles_part",
|
||||||
"build_for_scan", "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "language_of", "quick_part", "record",
|
"engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "quick_part", "record",
|
||||||
"ruleset_digest", "source_part", "verify_part",
|
"ruleset_digest", "source_part", "verify_part",
|
||||||
] # fmt: skip
|
] # fmt: skip
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ class FoldError(RuntimeError):
|
|||||||
class Fold:
|
class Fold:
|
||||||
"""One evaluation (or compile, or deployment) boundary: the groups evaluated, and what retrieval must not use."""
|
"""One evaluation (or compile, or deployment) boundary: the groups evaluated, and what retrieval must not use."""
|
||||||
|
|
||||||
kind: str # outer | leave_one_source_out | leave_one_framework_out | compile | deployment
|
kind: str # outer | compile | deployment
|
||||||
name: str
|
name: str
|
||||||
eval_groups: frozenset[str]
|
eval_groups: frozenset[str]
|
||||||
held_source: str | None = None
|
held_source: str | None = None
|
||||||
@@ -126,37 +126,6 @@ def outer_folds(examples: Sequence[Example], *, k: int = OUTER_K, seed: int = 0,
|
|||||||
return [Fold("outer", f"outer-{i}", frozenset(groups)) for i, groups in enumerate(assigned)]
|
return [Fold("outer", f"outer-{i}", frozenset(groups)) for i, groups in enumerate(assigned)]
|
||||||
|
|
||||||
|
|
||||||
def leave_one_source_out(examples: Sequence[Example], *, exclude: Iterable[str] = ()) -> list[Fold]:
|
|
||||||
excluded = set(exclude)
|
|
||||||
by_source: dict[str, set[str]] = {}
|
|
||||||
for example in examples:
|
|
||||||
if example.group not in excluded:
|
|
||||||
by_source.setdefault(example.source, set()).add(example.group)
|
|
||||||
return [Fold("leave_one_source_out", f"source-{s}", frozenset(g), held_source=s) for s, g in sorted(by_source.items())]
|
|
||||||
|
|
||||||
|
|
||||||
def leave_one_framework_out(
|
|
||||||
examples: Sequence[Example], *, exclude: Iterable[str] = (), min_groups: int = FRAMEWORK_MIN_GROUPS
|
|
||||||
) -> list[Fold]:
|
|
||||||
"""One fold per framework seen; below ``min_groups`` groups it is ``insufficient data`` and evaluates nothing."""
|
|
||||||
excluded = set(exclude)
|
|
||||||
by_framework: dict[str, set[str]] = {}
|
|
||||||
for example in examples:
|
|
||||||
for framework in example.frameworks:
|
|
||||||
if example.group not in excluded:
|
|
||||||
by_framework.setdefault(framework, set()).add(example.group)
|
|
||||||
folds = []
|
|
||||||
for framework, groups in sorted(by_framework.items()):
|
|
||||||
enough = len(groups) >= min_groups
|
|
||||||
folds.append(
|
|
||||||
Fold(
|
|
||||||
"leave_one_framework_out", f"framework-{framework}", frozenset(groups) if enough else frozenset(),
|
|
||||||
held_framework=framework, status="ok" if enough else INSUFFICIENT,
|
|
||||||
)
|
|
||||||
) # fmt: skip
|
|
||||||
return folds
|
|
||||||
|
|
||||||
|
|
||||||
def assert_disjoint(split: CompileSplit, folds: Iterable[Fold]) -> None:
|
def assert_disjoint(split: CompileSplit, folds: Iterable[Fold]) -> None:
|
||||||
"""The compile split shares no group with any evaluation fold; outer folds are pairwise disjoint."""
|
"""The compile split shares no group with any evaluation fold; outer folds are pairwise disjoint."""
|
||||||
seen: dict[str, str] = {}
|
seen: dict[str, str] = {}
|
||||||
@@ -181,6 +150,5 @@ def folds_digest(split: CompileSplit, folds: Iterable[Fold]) -> str:
|
|||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"COMPILE_FRACTION", "FRAMEWORK_MIN_GROUPS", "INSUFFICIENT", "OUTER_K", "CompileSplit", "Fold", "FoldError", "GroupInfo",
|
"COMPILE_FRACTION", "FRAMEWORK_MIN_GROUPS", "INSUFFICIENT", "OUTER_K", "CompileSplit", "Fold", "FoldError", "GroupInfo",
|
||||||
"assert_disjoint", "compile_split", "folds_digest", "group_table", "leave_one_framework_out", "leave_one_source_out",
|
"assert_disjoint", "compile_split", "folds_digest", "group_table", "outer_folds",
|
||||||
"outer_folds",
|
|
||||||
] # fmt: skip
|
] # fmt: skip
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ finding. A role's ``path``/``function`` are provenance, never features.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
|
||||||
import re
|
import re
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
@@ -536,13 +535,9 @@ def _confidence(role: Role) -> float:
|
|||||||
return round(1.0 / (role.depth + 1), 6)
|
return round(1.0 / (role.depth + 1), 6)
|
||||||
|
|
||||||
|
|
||||||
def load_roles(path: Path) -> RoleSet:
|
|
||||||
return RoleSet.from_json(json.loads(Path(path).read_text(encoding="utf-8")))
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"CWE_OPERATION", "DEPENDENCY_DIRS", "MAX_DEPTH", "MODEL_ROLE_KINDS", "OPERATION_CWE", "ROLE_KINDS", "SOURCE_KIND", "VERSION",
|
"CWE_OPERATION", "DEPENDENCY_DIRS", "MAX_DEPTH", "MODEL_ROLE_KINDS", "OPERATION_CWE", "ROLE_KINDS", "SOURCE_KIND", "VERSION",
|
||||||
"Function", "Role", "RoleSet", "base_roles", "body_roles", "dependency_files", "from_model_roles", "functions_of",
|
"Function", "Role", "RoleSet", "base_roles", "body_roles", "dependency_files", "from_model_roles", "functions_of",
|
||||||
"infer_for_checkout", "infer_wrappers", "language_key", "load_roles", "mentions", "origin_of", "parameters",
|
"infer_for_checkout", "infer_wrappers", "language_key", "mentions", "origin_of", "parameters",
|
||||||
"vocabulary_overlay",
|
"vocabulary_overlay",
|
||||||
] # fmt: skip
|
] # fmt: skip
|
||||||
|
|||||||
@@ -35,31 +35,6 @@ class StaticHint:
|
|||||||
evidence_candidate: bool = True
|
evidence_candidate: bool = True
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class MappingTaskRecord:
|
|
||||||
kind: MappingKind
|
|
||||||
path: str
|
|
||||||
rule_id: str
|
|
||||||
task: str
|
|
||||||
rationale: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class DifferentialMappingRecord:
|
|
||||||
path: str
|
|
||||||
change_kind: str
|
|
||||||
trust_boundary: str | None
|
|
||||||
rationale: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SharpEdgeRecord:
|
|
||||||
path: str
|
|
||||||
category: str
|
|
||||||
surface: str
|
|
||||||
rationale: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class EntryPointRecord:
|
class EntryPointRecord:
|
||||||
path: str
|
path: str
|
||||||
@@ -185,58 +160,6 @@ def attach_reachability_hints(targets: list[FileTarget], entry_points: list[Entr
|
|||||||
return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets]
|
return [replace(target, reachability_hints=hints_by_path.get(target.path, target.reachability_hints)) for target in targets]
|
||||||
|
|
||||||
|
|
||||||
def verifier_evidence_candidates(hints: list[StaticHint], *, path: str | None = None) -> list[dict[str, object]]:
|
|
||||||
selected = [hint for hint in hints if hint.evidence_candidate and (path is None or hint.path == path)]
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"source": hint.analyzer,
|
|
||||||
"rule_id": hint.rule_id,
|
|
||||||
"path": hint.path,
|
|
||||||
"line": hint.start_line,
|
|
||||||
"severity": hint.severity,
|
|
||||||
"message": hint.message,
|
|
||||||
"provenance": hint.provenance,
|
|
||||||
}
|
|
||||||
for hint in selected
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def semgrep_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
|
|
||||||
return [
|
|
||||||
MappingTaskRecord(
|
|
||||||
kind="semgrep",
|
|
||||||
path=hint.path,
|
|
||||||
rule_id=hint.rule_id,
|
|
||||||
task="pattern_variant_review",
|
|
||||||
rationale="Use Semgrep evidence to check variants and rule precision before verifier promotion.",
|
|
||||||
)
|
|
||||||
for hint in hints
|
|
||||||
if hint.analyzer == "semgrep"
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def codeql_mapping_tasks(hints: list[StaticHint]) -> list[MappingTaskRecord]:
|
|
||||||
return [
|
|
||||||
MappingTaskRecord(
|
|
||||||
kind="codeql",
|
|
||||||
path=hint.path,
|
|
||||||
rule_id=hint.rule_id,
|
|
||||||
task="source_sink_sanitizer_path_review",
|
|
||||||
rationale="Use CodeQL evidence to inspect source, sink, sanitizer, and path explanation.",
|
|
||||||
)
|
|
||||||
for hint in hints
|
|
||||||
if hint.analyzer == "codeql"
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def differential_mapping_record(path: str, *, change_kind: str, trust_boundary: str | None, rationale: str) -> DifferentialMappingRecord:
|
|
||||||
return DifferentialMappingRecord(path=path, change_kind=change_kind, trust_boundary=trust_boundary, rationale=rationale)
|
|
||||||
|
|
||||||
|
|
||||||
def sharp_edge_record(path: str, *, category: str, surface: str, rationale: str) -> SharpEdgeRecord:
|
|
||||||
return SharpEdgeRecord(path=path, category=category, surface=surface, rationale=rationale)
|
|
||||||
|
|
||||||
|
|
||||||
def write_static_hints(hints: list[StaticHint], path: Path) -> None:
|
def write_static_hints(hints: list[StaticHint], path: Path) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n")
|
path.write_text(json.dumps({"static_hints": [asdict(hint) for hint in hints]}, indent=2, sort_keys=True) + "\n")
|
||||||
|
|||||||
@@ -87,12 +87,6 @@ def verdicts(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def verdict(cpg: CpgResult, spec: ConfigSpec, *, function: str = "", file: str = "") -> Verdict | None:
|
|
||||||
"""The strongest single verdict, kept for callers that ask one question and want one answer."""
|
|
||||||
answers = verdicts(cpg, spec, function=function, file=file)
|
|
||||||
return answers[0] if answers else None
|
|
||||||
|
|
||||||
|
|
||||||
def _location(row: Mapping[str, object], file: str) -> str:
|
def _location(row: Mapping[str, object], file: str) -> str:
|
||||||
"""``path:line:function`` of the setting, when the region named a file.
|
"""``path:line:function`` of the setting, when the region named a file.
|
||||||
|
|
||||||
@@ -156,4 +150,4 @@ def _settings(rows: object, spec: ConfigSpec, *, function: str) -> list[dict[str
|
|||||||
return sorted(kept, key=lambda row: (str(row["method"]), _line(row), str(row["setting"])))
|
return sorted(kept, key=lambda row: (str(row["method"]), _line(row), str(row["setting"])))
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["request_params", "verdict", "verdicts"]
|
__all__ = ["request_params", "verdicts"]
|
||||||
|
|||||||
@@ -29,12 +29,10 @@ from typing import Any
|
|||||||
|
|
||||||
from ..cpg.backend import CpgResult
|
from ..cpg.backend import CpgResult
|
||||||
from ..redaction import redact_secrets
|
from ..redaction import redact_secrets
|
||||||
from .config_value import verdict as config_verdict
|
|
||||||
from .config_value import verdicts as config_verdicts
|
from .config_value import verdicts as config_verdicts
|
||||||
from .dominance import verdict as dominance_verdict
|
from .dominance import verdict as dominance_verdict
|
||||||
from .ladder import Rung, Verdict
|
from .ladder import Rung, Verdict
|
||||||
from .specs import ConfigSpec, DominanceSpec, TaintSpec
|
from .specs import ConfigSpec, DominanceSpec, TaintSpec
|
||||||
from .taint import verdict as taint_verdict
|
|
||||||
from .taint import verdicts as taint_verdicts
|
from .taint import verdicts as taint_verdicts
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -93,34 +91,11 @@ def _arbitrate_all(
|
|||||||
if isinstance(spec, ConfigSpec):
|
if isinstance(spec, ConfigSpec):
|
||||||
return config_verdicts(cpg, spec, function=function, file=path)
|
return config_verdicts(cpg, spec, function=function, file=path)
|
||||||
if isinstance(spec, DominanceSpec):
|
if isinstance(spec, DominanceSpec):
|
||||||
answer = _arbitrate(cpg, spec, path=path, function=function, parameter_sources=parameter_sources, call_depth=call_depth)
|
answer = dominance_verdict(cpg, spec, function=function, file=path)
|
||||||
return [answer] if answer is not None else []
|
return [answer] if answer is not None else []
|
||||||
return taint_verdicts(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
|
return taint_verdicts(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||||
|
|
||||||
|
|
||||||
def _arbitrate(
|
|
||||||
cpg: CpgResult,
|
|
||||||
spec: TaintSpec | DominanceSpec | ConfigSpec,
|
|
||||||
*,
|
|
||||||
path: str,
|
|
||||||
function: str,
|
|
||||||
parameter_sources: bool,
|
|
||||||
call_depth: int,
|
|
||||||
) -> Verdict | None:
|
|
||||||
"""Route a family to the arbiter that can actually decide it.
|
|
||||||
|
|
||||||
Dispatching on the spec type rather than on a family name keeps this honest: a family without an arbiter
|
|
||||||
cannot be silently routed to the wrong one. The pipeline first shipped taint-only, so access_control and
|
|
||||||
config_secrets -- whose arbiters had worked since group 4 -- produced nothing, and every one of their
|
|
||||||
pairs read as `both_silent` on the first end-to-end run.
|
|
||||||
"""
|
|
||||||
if isinstance(spec, DominanceSpec):
|
|
||||||
return dominance_verdict(cpg, spec, function=function, file=path)
|
|
||||||
if isinstance(spec, ConfigSpec):
|
|
||||||
return config_verdict(cpg, spec, function=function, file=path)
|
|
||||||
return taint_verdict(cpg, spec, function=function, file=path, parameter_sources=parameter_sources, call_depth=call_depth)
|
|
||||||
|
|
||||||
|
|
||||||
def scan_region(
|
def scan_region(
|
||||||
cpg: CpgResult,
|
cpg: CpgResult,
|
||||||
spec: TaintSpec | DominanceSpec | ConfigSpec,
|
spec: TaintSpec | DominanceSpec | ConfigSpec,
|
||||||
|
|||||||
@@ -1,87 +1,9 @@
|
|||||||
"""Reporting that cannot flatter itself (model-grounded-detection, Req 9.3, 11.1).
|
"""Report model coverage for the scanned regions."""
|
||||||
|
|
||||||
Three disciplines survive from the deleted publish/scoring layer, and each is here because its absence caused
|
|
||||||
a real error in this project's history rather than because it is tidy:
|
|
||||||
|
|
||||||
* **Per-slice rows, always.** A headline over mixed slices once hid that agent-written code scored 44.4%
|
|
||||||
while the hand-written development slice scored 83.3%. `render` refuses to produce anything without rows.
|
|
||||||
* **The overfitting gap beside the headline.** The distance between the development slice and the deciding
|
|
||||||
real-world one is stated, never left for the reader to compute. When a slice is missing the gap is
|
|
||||||
*absent*, not zero — reading a missing measurement as "no overfitting" is the flattering failure.
|
|
||||||
* **The rung on every finding.** A finding's strength is what established it; omitting the rung invites a
|
|
||||||
reader to treat a suspicion as a result.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
|
||||||
|
|
||||||
from ..findings import StaticFinding
|
__all__ = []
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SliceRow:
|
|
||||||
"""One slice's coverage. `covered` is pairs the model *split*, not pairs it flagged."""
|
|
||||||
|
|
||||||
slice: str
|
|
||||||
pairs: int
|
|
||||||
covered: int
|
|
||||||
|
|
||||||
@property
|
|
||||||
def rate(self) -> float:
|
|
||||||
return self.covered / self.pairs if self.pairs else 0.0
|
|
||||||
|
|
||||||
|
|
||||||
def overfitting_gap(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> float | None:
|
|
||||||
"""Development-slice coverage minus the deciding slice's, or ``None`` when either is absent.
|
|
||||||
|
|
||||||
``None`` rather than ``0.0`` on purpose: a slice that was never measured must not read as agreement
|
|
||||||
between them.
|
|
||||||
"""
|
|
||||||
by_name = {row.slice: row for row in rows}
|
|
||||||
if development not in by_name or deciding not in by_name:
|
|
||||||
return None
|
|
||||||
return by_name[development].rate - by_name[deciding].rate
|
|
||||||
|
|
||||||
|
|
||||||
def render(rows: Sequence[SliceRow], *, development: str = "", deciding: str = "") -> str:
|
|
||||||
"""The report. Raises when there are no per-slice rows, because a bare aggregate is the thing to prevent."""
|
|
||||||
if not rows:
|
|
||||||
raise ValueError("a report needs per-slice rows: there is no headline without them")
|
|
||||||
|
|
||||||
lines = ["## Model coverage", "", "| slice | pairs | covered | coverage |", "|---|---|---|---|"]
|
|
||||||
for row in sorted(rows, key=lambda r: r.slice):
|
|
||||||
lines.append(f"| {row.slice} | {row.pairs} | {row.covered} | {row.rate:.0%} |")
|
|
||||||
|
|
||||||
total_pairs = sum(row.pairs for row in rows)
|
|
||||||
total_covered = sum(row.covered for row in rows)
|
|
||||||
lines += [
|
|
||||||
"",
|
|
||||||
f"Across every slice: {total_covered}/{total_pairs} = {total_covered / total_pairs:.0%} — read the rows above it, not this line.",
|
|
||||||
]
|
|
||||||
|
|
||||||
gap = overfitting_gap(rows, development=development, deciding=deciding)
|
|
||||||
if gap is not None:
|
|
||||||
lines += [
|
|
||||||
"",
|
|
||||||
f"**Overfitting gap: {gap:+.0%}** — `{development}` (development) minus `{deciding}`, which is the deciding real-world slice.",
|
|
||||||
]
|
|
||||||
elif development or deciding:
|
|
||||||
lines += ["", f"**Overfitting gap: not measurable** — `{development}` or `{deciding}` is absent from this run."]
|
|
||||||
return "\n".join(lines) + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def render_findings(findings: Sequence[StaticFinding]) -> str:
|
|
||||||
"""Findings with the rung that established each one (Req 5.3)."""
|
|
||||||
lines = ["## Findings", "", "| finding | location | rung | severity |", "|---|---|---|---|"]
|
|
||||||
for finding in findings:
|
|
||||||
where = f"{finding.path}:{finding.line}" if finding.line else finding.path
|
|
||||||
lines.append(f"| {finding.title} | {where} | `{finding.rung}` | {finding.severity} |")
|
|
||||||
return "\n".join(lines) + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["SliceRow", "overfitting_gap", "render", "render_findings"]
|
|
||||||
|
|
||||||
|
|
||||||
def coverage_rows(regions: Sequence[object]) -> list[dict[str, object]]:
|
def coverage_rows(regions: Sequence[object]) -> list[dict[str, object]]:
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ graph establishes it and the LLM's claim is no longer load-bearing. A path a san
|
|||||||
`CORROBORATED`: the flow is real, but whether the sanitizer is *sufficient* is a semantic judgement the model
|
`CORROBORATED`: the flow is real, but whether the sanitizer is *sufficient* is a semantic judgement the model
|
||||||
cannot make, and that residual is the LLM's — checked against this verdict rather than averaged over runs.
|
cannot make, and that residual is the LLM's — checked against this verdict rather than averaged over runs.
|
||||||
|
|
||||||
No path at all yields `None`, not a negative verdict: the caller reports `suspicion`. And a query that failed
|
No path at all yields an empty list, not a negative verdict: the caller reports `suspicion`. And a query that failed
|
||||||
yields `None` too, because "the engine could not decide" is not "there is no flow" — conflating them is how a
|
yields `None` too, because "the engine could not decide" is not "there is no flow" — conflating them is how a
|
||||||
missing engine would silently become a clean bill of health.
|
missing engine would silently become a clean bill of health.
|
||||||
"""
|
"""
|
||||||
@@ -69,37 +69,6 @@ def request_params(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def verdict(
|
|
||||||
cpg: CpgResult, spec: TaintSpec, *, function: str = "", file: str = "", parameter_sources: bool = False, call_depth: int = 0
|
|
||||||
) -> Verdict | None:
|
|
||||||
"""The strongest verdict the taint query supports for ``spec`` in ``function``, or ``None`` to stay at suspicion.
|
|
||||||
|
|
||||||
``parameter_sources`` additionally treats the labeled function's own parameters as untrusted. That is right
|
|
||||||
for a function-level pair, where the function boundary *is* the trust boundary, and wrong for a whole
|
|
||||||
repository, where most parameters carry internal values — so it is off by default and the caller opts in.
|
|
||||||
An ENTRY POINT is the case where both are true at once: its parameters are attacker input by definition,
|
|
||||||
which is what the driver uses to decide.
|
|
||||||
|
|
||||||
``call_depth`` admits sinks that many call-graph levels below the labeled method. Zero keeps the old
|
|
||||||
function-local question; above zero is what lets a handler's parameter reach a sink in another module,
|
|
||||||
which is the shape VAmPI's SQL injection has and the shape a function-scoped question cannot see.
|
|
||||||
"""
|
|
||||||
# The strongest of the per-site verdicts, which is what this always meant. It used to run its own
|
|
||||||
# selection over the union of flows, and two copies of "which flow decides" is the duplication this
|
|
||||||
# module has been bitten by before: the per-site preference added to `verdicts` did not reach here, so
|
|
||||||
# the two entry points disagreed about the same rows.
|
|
||||||
#
|
|
||||||
# A flow the graph cannot see a discharge for is the finding; a sanitized or BOUNDED one is a question.
|
|
||||||
# libpng bounds `strcpy(outname+len, ".png")` with `(len = strlen(inname)) > 250` against a char[256],
|
|
||||||
# and entailing it is a claim the code contradicts. But the bound reaches the sink through an `error`
|
|
||||||
# flag rather than by dominating it, so the graph has not shown the copy is guarded either -- and a
|
|
||||||
# bound is no proof of safety in any case, off-by-one being the classic way they fail. Corroborated is
|
|
||||||
# the honest rung: the model cannot assert this, so the judge is asked instead of the site being
|
|
||||||
# silenced.
|
|
||||||
answers = verdicts(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
|
|
||||||
return answers[0] if answers else None
|
|
||||||
|
|
||||||
|
|
||||||
def _usable_flows(
|
def _usable_flows(
|
||||||
cpg: CpgResult,
|
cpg: CpgResult,
|
||||||
spec: TaintSpec,
|
spec: TaintSpec,
|
||||||
@@ -137,17 +106,7 @@ def verdicts(
|
|||||||
parameter_sources: bool = False,
|
parameter_sources: bool = False,
|
||||||
call_depth: int = 0,
|
call_depth: int = 0,
|
||||||
) -> list[Verdict]:
|
) -> list[Verdict]:
|
||||||
"""One verdict per distinct SINK SITE, strongest first.
|
"""One verdict per distinct sink site, strongest first."""
|
||||||
|
|
||||||
``verdict`` answers "what is the strongest thing to say about this region", which is the right question
|
|
||||||
for a pair: one labelled function, one bug, one answer. It is the wrong question for a region that spans
|
|
||||||
a file. A PHP file with an SQL injection on line 6 and a command injection on line 13 got ONE injection
|
|
||||||
verdict, and `system` won it on flow length -- so the SQL injection went unreported, by construction
|
|
||||||
rather than by any failure of the analysis.
|
|
||||||
|
|
||||||
The first element is exactly what ``verdict`` returns, so the pair path and the committed measurements
|
|
||||||
that rest on it are unchanged.
|
|
||||||
"""
|
|
||||||
flows = _usable_flows(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
|
flows = _usable_flows(cpg, spec, function=function, file=file, parameter_sources=parameter_sources, call_depth=call_depth)
|
||||||
if not flows:
|
if not flows:
|
||||||
return []
|
return []
|
||||||
@@ -164,7 +123,7 @@ def verdicts(
|
|||||||
preferred = [flow for flow in site_flows if _from_modelled_source(flow, spec)] or list(site_flows)
|
preferred = [flow for flow in site_flows if _from_modelled_source(flow, spec)] or list(site_flows)
|
||||||
open_flows = [flow for flow in preferred if not flow["sanitized"] and not flow["bounded"]]
|
open_flows = [flow for flow in preferred if not flow["sanitized"] and not flow["bounded"]]
|
||||||
chosen.append(min(open_flows or preferred, key=lambda flow: _rank(flow, spec)))
|
chosen.append(min(open_flows or preferred, key=lambda flow: _rank(flow, spec)))
|
||||||
# Same total order the single-verdict path uses, so `verdicts(...)[0] == verdict(...)`.
|
# Sort strongest first, then by site for stable results.
|
||||||
chosen.sort(key=lambda flow: (bool(flow["sanitized"]) or bool(flow["bounded"]), _rank(flow, spec)))
|
chosen.sort(key=lambda flow: (bool(flow["sanitized"]) or bool(flow["bounded"]), _rank(flow, spec)))
|
||||||
return [_verdict_for(flow, spec) for flow in chosen]
|
return [_verdict_for(flow, spec) for flow in chosen]
|
||||||
|
|
||||||
@@ -309,4 +268,4 @@ def _location(flow: Mapping[str, object]) -> str:
|
|||||||
return f"{where}:{line}:{flow.get('sinkMethod') or '?'}"
|
return f"{where}:{line}:{flow.get('sinkMethod') or '?'}"
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["request_params", "verdict"]
|
__all__ = ["request_params", "verdicts"]
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ from __future__ import annotations
|
|||||||
import tomllib
|
import tomllib
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
FAMILY_IDS = (
|
FAMILY_IDS = (
|
||||||
"injection",
|
"injection",
|
||||||
@@ -28,7 +27,6 @@ FAMILY_IDS = (
|
|||||||
"unknown",
|
"unknown",
|
||||||
)
|
)
|
||||||
VERIFIER_KINDS = ("canary", "static", "none")
|
VERIFIER_KINDS = ("canary", "static", "none")
|
||||||
Relation = Literal["same", "parent_child", "lateral", "fabricated"]
|
|
||||||
DEFAULT_FAMILIES_PATH = Path(__file__).resolve().parents[1] / "ruleset" / "families.toml"
|
DEFAULT_FAMILIES_PATH = Path(__file__).resolve().parents[1] / "ruleset" / "families.toml"
|
||||||
|
|
||||||
_FAMILY_FIELDS = frozenset({"id", "description", "cwes", "mechanisms", "verifier", "parent", "limits"})
|
_FAMILY_FIELDS = frozenset({"id", "description", "cwes", "mechanisms", "verifier", "parent", "limits"})
|
||||||
@@ -72,24 +70,6 @@ class FamilyTaxonomy:
|
|||||||
def family_of_mechanism(self, mechanism: str) -> Family | None:
|
def family_of_mechanism(self, mechanism: str) -> Family | None:
|
||||||
return next((family for family in self.families if mechanism in family.mechanisms), None)
|
return next((family for family in self.families if mechanism in family.mechanisms), None)
|
||||||
|
|
||||||
@property
|
|
||||||
def hierarchical(self) -> bool:
|
|
||||||
"""True when some family declares a parent. While this is False, partial credit can never be earned."""
|
|
||||||
return any(family.parent for family in self.families)
|
|
||||||
|
|
||||||
def related(self, left: str, right: str) -> Relation:
|
|
||||||
"""How two family answers relate. Abstention is not partial credit: ``unknown`` against a real
|
|
||||||
family is ``lateral``, never ``parent_child``."""
|
|
||||||
known = {family.id: family for family in self.families}
|
|
||||||
if left not in known or right not in known:
|
|
||||||
return "fabricated"
|
|
||||||
if left == right:
|
|
||||||
return "same"
|
|
||||||
if known[left].parent == right or known[right].parent == left:
|
|
||||||
return "parent_child"
|
|
||||||
return "lateral"
|
|
||||||
|
|
||||||
|
|
||||||
def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
|
def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
|
||||||
"""Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file);
|
"""Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file);
|
||||||
a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``."""
|
a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``."""
|
||||||
@@ -189,7 +169,6 @@ __all__ = [
|
|||||||
"FamiliesError",
|
"FamiliesError",
|
||||||
"Family",
|
"Family",
|
||||||
"FamilyTaxonomy",
|
"FamilyTaxonomy",
|
||||||
"Relation",
|
|
||||||
"load_families",
|
"load_families",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
|
|||||||
runs/<run>/<task>/output.tar the runner's PUT
|
runs/<run>/<task>/output.tar the runner's PUT
|
||||||
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
|
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
|
||||||
runs/<run>/<task>/done.json status, bytes, file count, finished
|
runs/<run>/<task>/done.json status, bytes, file count, finished
|
||||||
runs/<run>/inputs/<name> an input no task produces (``put_input``)
|
|
||||||
|
|
||||||
Nothing here logs or prints a URL.
|
Nothing here logs or prints a URL.
|
||||||
"""
|
"""
|
||||||
@@ -164,11 +163,6 @@ class Delivery:
|
|||||||
put.append(artifact)
|
put.append(artifact)
|
||||||
return put
|
return put
|
||||||
|
|
||||||
def put_input(self, name: str, path: Path) -> str:
|
|
||||||
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
|
|
||||||
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
|
|
||||||
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
|
|
||||||
|
|
||||||
# --- state ---------------------------------------------------------------------------------------------------
|
# --- state ---------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
def save_state(self, data: Mapping[str, Any]) -> None:
|
def save_state(self, data: Mapping[str, Any]) -> None:
|
||||||
|
|||||||
@@ -1,61 +1,29 @@
|
|||||||
"""Per-task network egress on Agent Substrate (ai-service-plane, task 0: gateway + receiver Service).
|
"""Per-task TLS egress to the store, Git hosts and model endpoints.
|
||||||
|
|
||||||
Every TCP connection an actor opens (DNS aside) is redirected to Substrate's egress gateway
|
The gateway denies by default. Egress writes and checks each actor's policy before resume,
|
||||||
(``atenet-egress.ate-system``), which denies by default: an actor without an ``EgressPolicy`` gets no tunnel, and
|
then waits for the cached denial to expire. Deleting the actor removes its policy.
|
||||||
a refused connection is accepted and then closed. :func:`policy_for` grants a task exactly what it needs:
|
|
||||||
|
|
||||||
* ``http`` on port 80 to the artifact receiver's cluster name (``ousast-receiver.ax-system.svc.cluster.local``, a
|
|
||||||
selector-less Service whose endpoint is this host's receiver, ``ops/ax/receiver-service.yaml.tmpl``). The actor
|
|
||||||
cannot resolve cluster names, so the runner dials ``OUSAST_ARTIFACT_DIAL`` (the Service's ClusterIP, port 80)
|
|
||||||
with the receiver's name as ``Host``. The gateway decides the policy on that ``Host`` but connects to the
|
|
||||||
address the actor dialled (measured live: agentgateway's upstream is the CONNECT authority, not the Host), so
|
|
||||||
the dial address must be one the gateway pod reaches: the ClusterIP, which kube-proxy maps to the host.
|
|
||||||
* ``tls_passthrough`` on 443 for the Git hosts of the bound Workspaces' ``https`` repos (smart HTTP talks to the
|
|
||||||
repo's host only) and for the bound Model's ``openultrasast.io/egress-hosts`` (``api.deepseek.com``).
|
|
||||||
|
|
||||||
Nothing else: no wildcard, no IP address (the gateway rejects them), no ``https`` rule (that one intercepts TLS and
|
|
||||||
the actor would have to trust the gateway CA, which ax templates do not mount). :class:`Egress` writes the policy
|
|
||||||
with ``kubectl ate create egress-policy`` after ``ax apply`` created the actor, reads it back, and before resume
|
|
||||||
waits ``OUSAST_EGRESS_SETTLE`` s (11) because the gateway caches a missing policy as deny for 10 s. The kubectl-ate
|
|
||||||
CLI has no delete verb: Substrate's store deletes the policy with its actor (``ON DELETE CASCADE``), so
|
|
||||||
:meth:`Egress.delete` runs after ``ax delete task`` and confirms the policy is gone.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import tarfile
|
|
||||||
import threading
|
|
||||||
import time
|
import time
|
||||||
from collections.abc import Iterable, Mapping
|
from collections.abc import Iterable, Mapping
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import unquote, urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from .manifests import Model, Task, Workspace
|
from .manifests import Model, Task, Workspace
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"INPUTS_PATH",
|
|
||||||
"RECEIVER_HOST",
|
|
||||||
"RECEIVER_PORT",
|
|
||||||
"Egress",
|
"Egress",
|
||||||
"Receiver",
|
|
||||||
"git_hosts",
|
"git_hosts",
|
||||||
"receiver_cluster_ip",
|
|
||||||
"policy_for",
|
"policy_for",
|
||||||
"receiver_address",
|
|
||||||
"rules_of",
|
"rules_of",
|
||||||
]
|
]
|
||||||
|
|
||||||
RECEIVER_HOST = "ousast-receiver.ax-system.svc.cluster.local"
|
|
||||||
INPUTS_PATH = "/inputs/" # GET <receiver>/inputs/<producer>/<artifact>: a consumer fetches a declared input
|
|
||||||
RECEIVER_PORT = 18090 # the receiver Service's targetPort; ops/ax/up.sh renders the Service with the same number
|
|
||||||
_NOT_FOUND = ("not found", "notfound")
|
_NOT_FOUND = ("not found", "notfound")
|
||||||
|
|
||||||
|
|
||||||
@@ -89,12 +57,10 @@ def policy_for(
|
|||||||
workspaces: Iterable[Workspace],
|
workspaces: Iterable[Workspace],
|
||||||
model: Model | None,
|
model: Model | None,
|
||||||
store_host: str | None,
|
store_host: str | None,
|
||||||
receiver_host: str | None = None,
|
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
|
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
|
||||||
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
|
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
|
||||||
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless
|
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. An http Git repo cannot be reached under
|
||||||
``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under
|
|
||||||
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
|
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
|
||||||
hostnames only)."""
|
hostnames only)."""
|
||||||
tls, plain = git_hosts(workspaces)
|
tls, plain = git_hosts(workspaces)
|
||||||
@@ -106,8 +72,6 @@ def policy_for(
|
|||||||
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
|
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
|
||||||
hosts.add(store_host.lower())
|
hosts.add(store_host.lower())
|
||||||
rules: list[dict[str, Any]] = []
|
rules: list[dict[str, Any]] = []
|
||||||
if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it)
|
|
||||||
rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}})
|
|
||||||
if hosts:
|
if hosts:
|
||||||
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
|
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
|
||||||
return {"rules": rules}
|
return {"rules": rules}
|
||||||
@@ -125,41 +89,6 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|||||||
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
|
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
def receiver_cluster_ip(context: str) -> str | None:
|
|
||||||
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when
|
|
||||||
kubectl cannot say."""
|
|
||||||
if not context:
|
|
||||||
raise RuntimeError("receiver_cluster_ip needs the profile's kube context")
|
|
||||||
argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"]
|
|
||||||
try:
|
|
||||||
proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20)
|
|
||||||
except (OSError, subprocess.TimeoutExpired):
|
|
||||||
return None
|
|
||||||
address = proc.stdout.strip()
|
|
||||||
return address if proc.returncode == 0 and _is_ip(address) else None
|
|
||||||
|
|
||||||
|
|
||||||
def receiver_address(port: int, context: str | None) -> tuple[str, str | None]:
|
|
||||||
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the
|
|
||||||
profile's kube context the Service's ClusterIP is looked up in.
|
|
||||||
|
|
||||||
``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://<host>:<port>/`` and no dial (tests, a host the actor
|
|
||||||
reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the
|
|
||||||
Service's ClusterIP on port 80: the gateway checks the policy against ``Host`` but connects to the address the
|
|
||||||
actor dialled (its CONNECT authority), so that address must reach the receiver from the gateway pod.
|
|
||||||
"""
|
|
||||||
host = os.environ.get("OUSAST_ARTIFACT_HOST")
|
|
||||||
if host:
|
|
||||||
return f"http://{host}:{port}/", None
|
|
||||||
dial = os.environ.get("OUSAST_ARTIFACT_DIAL")
|
|
||||||
if not dial:
|
|
||||||
if not context:
|
|
||||||
raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up")
|
|
||||||
address = receiver_cluster_ip(context)
|
|
||||||
dial = f"{address}:80" if address else None
|
|
||||||
return f"http://{RECEIVER_HOST}/", dial
|
|
||||||
|
|
||||||
|
|
||||||
class Egress:
|
class Egress:
|
||||||
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
|
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
|
||||||
|
|
||||||
@@ -230,95 +159,3 @@ class Egress:
|
|||||||
time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11))
|
time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11))
|
||||||
|
|
||||||
|
|
||||||
class Receiver(ThreadingHTTPServer):
|
|
||||||
"""Accepts one tar per running task and extracts it under ``<run dir>/<task>/`` (POST); serves a running task
|
|
||||||
the producer artifacts it declared as inputs (``GET /inputs/<producer>/<artifact>``, ``inputs[task]``)."""
|
|
||||||
|
|
||||||
daemon_threads = True
|
|
||||||
|
|
||||||
def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None:
|
|
||||||
super().__init__(("0.0.0.0", port), _Handler)
|
|
||||||
self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]()
|
|
||||||
self.inputs: dict[str, frozenset[str]] = {}
|
|
||||||
self.lock = threading.Lock()
|
|
||||||
self.url, self.dial = receiver_address(self.server_address[1], context)
|
|
||||||
|
|
||||||
def _task(self, headers: Any) -> str | None:
|
|
||||||
"""The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None."""
|
|
||||||
task = str(headers.get("X-Ousast-Task", ""))
|
|
||||||
with self.lock:
|
|
||||||
return task if headers.get("X-Ousast-Run") == self.run_name and task in self.running else None
|
|
||||||
|
|
||||||
def input_file(self, headers: Any, ref: str) -> tuple[int, str | Path]:
|
|
||||||
"""(200, the file) when ``ref`` is one of the requesting task's declared inputs, else (403/404, why).
|
|
||||||
|
|
||||||
``ref`` must equal a declared ``<producer>/<artifact>`` exactly and resolve to a regular file under the run
|
|
||||||
directory; nothing is normalised first, so ``..`` or an absolute path never names a declared input."""
|
|
||||||
task = self._task(headers)
|
|
||||||
if task is None:
|
|
||||||
return 403, "unknown run or task"
|
|
||||||
with self.lock:
|
|
||||||
declared = ref in self.inputs.get(task, frozenset())
|
|
||||||
if not declared:
|
|
||||||
return 403, f"{ref!r} is not an input of {task}"
|
|
||||||
root = self.base.resolve()
|
|
||||||
path = (root / ref).resolve()
|
|
||||||
if not path.is_relative_to(root) or not path.is_file():
|
|
||||||
return 404, f"{ref} was not delivered"
|
|
||||||
return 200, path
|
|
||||||
|
|
||||||
def accept(self, headers: Any, body: bytes) -> tuple[int, str]:
|
|
||||||
task = self._task(headers)
|
|
||||||
if task is None:
|
|
||||||
return 403, "unknown run or task"
|
|
||||||
if not str(headers.get("Content-Type", "")).startswith("application/x-tar"):
|
|
||||||
return 415, "expected application/x-tar"
|
|
||||||
dest = (self.base / task).resolve()
|
|
||||||
try:
|
|
||||||
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as tar:
|
|
||||||
members = tar.getmembers()
|
|
||||||
for m in members:
|
|
||||||
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
|
|
||||||
return 400, f"rejected member {m.name}"
|
|
||||||
dest.mkdir(parents=True, exist_ok=True)
|
|
||||||
tar.extractall(dest, members) # every member checked above
|
|
||||||
except tarfile.TarError as exc:
|
|
||||||
return 400, f"bad tar: {exc}"
|
|
||||||
with self.lock:
|
|
||||||
self.delivered.add(task)
|
|
||||||
return 200, "ok"
|
|
||||||
|
|
||||||
|
|
||||||
class _Handler(BaseHTTPRequestHandler):
|
|
||||||
def do_GET(self) -> None: # noqa: N802 - http.server's naming
|
|
||||||
server: Receiver = self.server # type: ignore[assignment]
|
|
||||||
path = urlsplit(self.path).path
|
|
||||||
if not path.startswith(INPUTS_PATH):
|
|
||||||
self._text(404, "not found")
|
|
||||||
return
|
|
||||||
code, found = server.input_file(self.headers, unquote(path[len(INPUTS_PATH) :]))
|
|
||||||
if not isinstance(found, Path):
|
|
||||||
self._text(code, found)
|
|
||||||
return
|
|
||||||
with found.open("rb") as source: # streamed: a producer artifact can be far larger than any env value
|
|
||||||
self.send_response(200)
|
|
||||||
self.send_header("Content-Type", "application/octet-stream")
|
|
||||||
self.send_header("Content-Length", str(os.fstat(source.fileno()).st_size))
|
|
||||||
self.end_headers()
|
|
||||||
shutil.copyfileobj(source, self.wfile, 1 << 16)
|
|
||||||
|
|
||||||
def do_POST(self) -> None: # noqa: N802 - http.server's naming
|
|
||||||
server: Receiver = self.server # type: ignore[assignment]
|
|
||||||
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
|
|
||||||
self._text(*server.accept(self.headers, body))
|
|
||||||
|
|
||||||
def _text(self, code: int, text: str) -> None:
|
|
||||||
data = text.encode()
|
|
||||||
self.send_response(code)
|
|
||||||
self.send_header("Content-Type", "text/plain")
|
|
||||||
self.send_header("Content-Length", str(len(data)))
|
|
||||||
self.end_headers()
|
|
||||||
self.wfile.write(data)
|
|
||||||
|
|
||||||
def log_message(self, format: str, *args: object) -> None:
|
|
||||||
return
|
|
||||||
|
|||||||
@@ -10,8 +10,7 @@ credential from this process's environment, never from a file (Req 4.6), and the
|
|||||||
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
|
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
|
||||||
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
|
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
|
||||||
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
|
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
|
||||||
delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind
|
delivery fail it; ``ax delete task`` follows.
|
||||||
profile for group 2 only (task 2.6 removes it).
|
|
||||||
|
|
||||||
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
|
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
|
||||||
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
|
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
|
||||||
@@ -40,13 +39,12 @@ from dataclasses import dataclass, field
|
|||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urlsplit
|
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from .delivery import Delivery, DeliveryError, declared
|
from .delivery import Delivery, DeliveryError, declared
|
||||||
from .doctor import doctor
|
from .doctor import doctor
|
||||||
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for
|
from .egress import Egress, policy_for
|
||||||
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
|
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
|
||||||
from .memory import open_store
|
from .memory import open_store
|
||||||
from .profile import PlaneProfile, load_profile
|
from .profile import PlaneProfile, load_profile
|
||||||
@@ -142,16 +140,14 @@ def _model_doc(model: Model) -> dict[str, Any]:
|
|||||||
return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
|
return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
|
||||||
|
|
||||||
|
|
||||||
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]:
|
def render_task(run: Run, entry: RunTask, manifests: Manifests) -> list[dict[str, Any]]:
|
||||||
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
|
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
|
||||||
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
|
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
|
||||||
(``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given."""
|
(``OUSAST_INPUTS``) after the start by presigned URL."""
|
||||||
task: Task = manifests.tasks[entry.task]
|
task: Task = manifests.tasks[entry.task]
|
||||||
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
|
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
|
||||||
env = {e.name: e.value for e in task.env}
|
env = {e.name: e.value for e in task.env}
|
||||||
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
|
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
|
||||||
if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env
|
|
||||||
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
|
|
||||||
budget = _fields(entry.budget) if entry.budget else {}
|
budget = _fields(entry.budget) if entry.budget else {}
|
||||||
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
|
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
|
||||||
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
|
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
|
||||||
@@ -341,25 +337,22 @@ def _execute(
|
|||||||
base: Path,
|
base: Path,
|
||||||
delivery: Delivery,
|
delivery: Delivery,
|
||||||
router: Router,
|
router: Router,
|
||||||
receiver: Receiver | None = None,
|
|
||||||
) -> tuple[str, str]:
|
) -> tuple[str, str]:
|
||||||
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
|
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
|
||||||
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
|
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
|
||||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None)
|
docs = render_task(run, entry, manifests)
|
||||||
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
|
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
|
||||||
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
|
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
|
||||||
workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
|
workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
|
||||||
body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
body = delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
||||||
store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None)
|
|
||||||
if receiver is None:
|
|
||||||
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
||||||
|
|
||||||
def delivered() -> bool:
|
def delivered() -> bool:
|
||||||
return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name)
|
return delivery.delivered(entry.name)
|
||||||
|
|
||||||
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
|
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
|
||||||
policy = policy_for(task, workspaces, model, store_host, receiver_host)
|
policy = policy_for(task, workspaces, model, delivery.host)
|
||||||
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
|
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
|
||||||
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
|
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
|
||||||
egress.apply(ax_name, atespace, policy)
|
egress.apply(ax_name, atespace, policy)
|
||||||
@@ -369,7 +362,7 @@ def _execute(
|
|||||||
return "failed", str(exc)
|
return "failed", str(exc)
|
||||||
try:
|
try:
|
||||||
failure = _await(ax, ax_name, delivered, start)
|
failure = _await(ax, ax_name, delivered, start)
|
||||||
if failure is None and receiver is None:
|
if failure is None:
|
||||||
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
|
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
|
||||||
except DeliveryError as exc:
|
except DeliveryError as exc:
|
||||||
failure = str(exc)
|
failure = str(exc)
|
||||||
@@ -408,11 +401,6 @@ def run(
|
|||||||
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
|
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
|
||||||
mirror=delivery.save_state,
|
mirror=delivery.save_state,
|
||||||
)
|
)
|
||||||
receiver: Receiver | None = None
|
|
||||||
if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it)
|
|
||||||
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
|
|
||||||
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
|
|
||||||
threading.Thread(target=receiver.serve_forever, daemon=True).start()
|
|
||||||
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
|
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
|
||||||
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
|
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
|
||||||
for entry in run_spec.tasks:
|
for entry in run_spec.tasks:
|
||||||
@@ -427,7 +415,7 @@ def run(
|
|||||||
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
|
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
|
||||||
state.set(entry.name, status="running", started=_now(), finished=None)
|
state.set(entry.name, status="running", started=_now(), finished=None)
|
||||||
try:
|
try:
|
||||||
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver)
|
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router)
|
||||||
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
|
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
|
||||||
outcome = ("failed", f"{type(exc).__name__}: {exc}")
|
outcome = ("failed", f"{type(exc).__name__}: {exc}")
|
||||||
finished.put((entry.name, *outcome))
|
finished.put((entry.name, *outcome))
|
||||||
@@ -444,27 +432,17 @@ def run(
|
|||||||
pending.remove(entry)
|
pending.remove(entry)
|
||||||
running[entry.name] = entry
|
running[entry.name] = entry
|
||||||
blocked.add(entry.serialize or "")
|
blocked.add(entry.serialize or "")
|
||||||
if receiver is not None:
|
|
||||||
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
|
|
||||||
receiver.running.add(entry.name)
|
|
||||||
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
|
|
||||||
threading.Thread(target=worker, args=(entry,), daemon=True).start()
|
threading.Thread(target=worker, args=(entry,), daemon=True).start()
|
||||||
if not running:
|
if not running:
|
||||||
break # nothing can start: a producer stopped short
|
break # nothing can start: a producer stopped short
|
||||||
name, status, reason = finished.get()
|
name, status, reason = finished.get()
|
||||||
del running[name]
|
del running[name]
|
||||||
if receiver is not None:
|
|
||||||
with receiver.lock:
|
|
||||||
receiver.running.discard(name)
|
|
||||||
receiver.inputs.pop(name, None)
|
|
||||||
summary = _read_json(base / name / "summary.json") or {}
|
summary = _read_json(base / name / "summary.json") or {}
|
||||||
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
|
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
|
||||||
halted = halted or status == "failed"
|
halted = halted or status == "failed"
|
||||||
if status == "unfinished":
|
if status == "unfinished":
|
||||||
pending = [t for t in pending if name not in t.producers]
|
pending = [t for t in pending if name not in t.producers]
|
||||||
finally:
|
finally:
|
||||||
if receiver is not None:
|
|
||||||
receiver.shutdown()
|
|
||||||
lock.unlink(missing_ok=True)
|
lock.unlink(missing_ok=True)
|
||||||
statuses = [state.status(t.name) for t in run_spec.tasks]
|
statuses = [state.status(t.name) for t in run_spec.tasks]
|
||||||
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
|
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
|
||||||
@@ -546,4 +524,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
__all__ = ["Ax", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||||
|
|||||||
@@ -25,8 +25,7 @@ The command runs as a child process, ``python -m openultrasast.plane.tasks.<spec
|
|||||||
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
|
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
|
||||||
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
|
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
|
||||||
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
|
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
|
||||||
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` (or POSTs it to ``OUSAST_ARTIFACT_URL``, the receiver
|
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` -- that delivery is the task's
|
||||||
path kept until task 2.6 removes it) -- that delivery is the task's
|
|
||||||
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
|
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
|
||||||
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
|
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
|
||||||
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
|
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
|
||||||
@@ -48,13 +47,9 @@ Environment the runner reads, all optional except the two ax variables:
|
|||||||
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
|
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
|
||||||
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
|
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
|
||||||
OUSAST_GIT_PINS JSON ``{"<workspace>/<git name>": "<40-hex commit>"}`` from the reconciler
|
OUSAST_GIT_PINS JSON ``{"<workspace>/<git name>": "<40-hex commit>"}`` from the reconciler
|
||||||
OUSAST_ARTIFACT_URL, OUSAST_RUN where to deliver, and the run name carried in ``X-Ousast-Run``
|
|
||||||
OUSAST_ARTIFACT_DIAL ``host:port`` to connect to instead of the URL's host, which is sent as
|
|
||||||
``Host`` (the egress gateway decides on and resolves that name)
|
|
||||||
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
|
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
|
||||||
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
|
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
|
||||||
command each is fetched from ``delivery.inputs[NAME]`` (else from
|
command each is fetched from ``delivery.inputs[NAME]``
|
||||||
``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` with the receiver's dial and headers)
|
|
||||||
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
|
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
|
||||||
fetch delivers a failed summary naming the input, the command never runs
|
fetch delivers a failed summary naming the input, the command never runs
|
||||||
|
|
||||||
@@ -700,16 +695,6 @@ def build_tar(output_dir: Path) -> bytes:
|
|||||||
return buffer.getvalue()
|
return buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[str, dict[str, str]]:
|
|
||||||
"""(URL to connect to, headers): with ``dial`` the connection goes there and the URL's host is sent as ``Host``."""
|
|
||||||
out = dict(headers)
|
|
||||||
if not dial:
|
|
||||||
return url, out
|
|
||||||
parts = urlsplit(url)
|
|
||||||
out["Host"] = parts.netloc
|
|
||||||
return parts._replace(netloc=dial).geturl(), out
|
|
||||||
|
|
||||||
|
|
||||||
def safe_url(url: str) -> str:
|
def safe_url(url: str) -> str:
|
||||||
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
|
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
|
||||||
parts = urlsplit(url)
|
parts = urlsplit(url)
|
||||||
@@ -723,18 +708,14 @@ def deliver(
|
|||||||
*,
|
*,
|
||||||
attempts: int = DELIVERY_ATTEMPTS,
|
attempts: int = DELIVERY_ATTEMPTS,
|
||||||
backoff: float = 1.0,
|
backoff: float = 1.0,
|
||||||
dial: str | None = None,
|
|
||||||
method: str = "POST",
|
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry
|
"""PUT the payload with retries and doubling backoff; return the 2xx status.
|
||||||
with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query.
|
|
||||||
|
|
||||||
``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it
|
Log lines and errors carry the URL without its query.
|
||||||
cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself.
|
|
||||||
"""
|
"""
|
||||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
last = "no attempt made"
|
||||||
for attempt in range(1, attempts + 1):
|
for attempt in range(1, attempts + 1):
|
||||||
request = urllib.request.Request(target, data=payload, method=method, headers=headers)
|
request = urllib.request.Request(url, data=payload, method="PUT", headers=headers)
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
|
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
|
||||||
status = int(response.status)
|
status = int(response.status)
|
||||||
@@ -752,16 +733,16 @@ def deliver(
|
|||||||
|
|
||||||
|
|
||||||
def fetch(
|
def fetch(
|
||||||
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0, dial: str | None = None
|
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0
|
||||||
) -> int:
|
) -> int:
|
||||||
"""GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`.
|
"""GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`.
|
||||||
|
|
||||||
Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed."""
|
Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed."""
|
||||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
last = "no attempt made"
|
||||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
partial = dest.with_name(dest.name + ".part")
|
partial = dest.with_name(dest.name + ".part")
|
||||||
for attempt in range(1, attempts + 1):
|
for attempt in range(1, attempts + 1):
|
||||||
request = urllib.request.Request(target, method="GET", headers=headers)
|
request = urllib.request.Request(url, method="GET", headers=headers)
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310
|
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310
|
||||||
shutil.copyfileobj(response, out, 1 << 16)
|
shutil.copyfileobj(response, out, 1 << 16)
|
||||||
@@ -784,7 +765,7 @@ def fetch(
|
|||||||
|
|
||||||
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
|
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
|
||||||
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
|
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
|
||||||
presigned GET, no headers) when the start request carries one, else from the receiver.
|
presigned GET, no headers) carried by the start request.
|
||||||
|
|
||||||
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
|
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
|
||||||
raw = environ.get("OUSAST_INPUTS")
|
raw = environ.get("OUSAST_INPUTS")
|
||||||
@@ -796,21 +777,15 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
|
|||||||
raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None
|
raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None
|
||||||
if not isinstance(inputs, dict):
|
if not isinstance(inputs, dict):
|
||||||
raise RunnerError("OUSAST_INPUTS must map input names to <producer>/<artifact>")
|
raise RunnerError("OUSAST_INPUTS must map input names to <producer>/<artifact>")
|
||||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
|
||||||
headers = {"X-Ousast-Run": environ.get("OUSAST_RUN", ""), "X-Ousast-Task": environ.get("OUSAST_TASK", "")}
|
|
||||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||||
fetched: list[Path] = []
|
fetched: list[Path] = []
|
||||||
for name, ref in sorted(inputs.items()):
|
for name, ref in sorted(inputs.items()):
|
||||||
dest = environ.get(f"OUSAST_INPUT_{name}")
|
dest = environ.get(f"OUSAST_INPUT_{name}")
|
||||||
presigned = delivery.inputs.get(name) if delivery is not None else None
|
presigned = delivery.inputs.get(name) if delivery is not None else None
|
||||||
if not dest or not isinstance(ref, str) or not (presigned or url):
|
if not dest or not isinstance(ref, str) or not presigned:
|
||||||
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}")
|
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL or OUSAST_INPUT_{name}")
|
||||||
try:
|
try:
|
||||||
if presigned:
|
|
||||||
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
||||||
else:
|
|
||||||
source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
|
|
||||||
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
|
||||||
except DeliveryError as exc:
|
except DeliveryError as exc:
|
||||||
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
|
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
|
||||||
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
|
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
|
||||||
@@ -818,23 +793,14 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
|
|||||||
return fetched
|
return fetched
|
||||||
|
|
||||||
|
|
||||||
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
def _deliver_output(output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
||||||
"""PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver;
|
"""PUT the output tar to ``delivery.put`` when the start request carried one;
|
||||||
False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
|
False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
|
||||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||||
if delivery is not None and delivery.put:
|
if delivery is not None and delivery.put:
|
||||||
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT")
|
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff)
|
||||||
return True
|
return True
|
||||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
|
||||||
if not url:
|
|
||||||
return False
|
return False
|
||||||
# the Run's task name (``OUSAST_TASK``) is what the receiver knows; the ax Task is named ``<run>-<task>``
|
|
||||||
headers = {"Content-Type": "application/x-tar", "X-Ousast-Task": environ.get("OUSAST_TASK") or task.metadata.name}
|
|
||||||
run = environ.get("OUSAST_RUN")
|
|
||||||
if run:
|
|
||||||
headers["X-Ousast-Run"] = run
|
|
||||||
deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
# --- entrypoint --------------------------------------------------------------------------------------------------
|
# --- entrypoint --------------------------------------------------------------------------------------------------
|
||||||
@@ -849,7 +815,7 @@ class _Session:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def marker(self) -> Path:
|
def marker(self) -> Path:
|
||||||
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the receiver has it."""
|
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the store has it."""
|
||||||
return self.state_dir / f"{self.task.metadata.name}.done.json"
|
return self.state_dir / f"{self.task.metadata.name}.done.json"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -893,7 +859,7 @@ def _finish(
|
|||||||
"""
|
"""
|
||||||
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
|
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
|
||||||
try:
|
try:
|
||||||
marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery)
|
marker["delivered"] = _deliver_output(session.output_dir, env, delivery)
|
||||||
except DeliveryError as exc:
|
except DeliveryError as exc:
|
||||||
log.error("artifact delivery failed: %s", exc)
|
log.error("artifact delivery failed: %s", exc)
|
||||||
marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
|
marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
|
||||||
@@ -938,7 +904,7 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
|
|||||||
return 0
|
return 0
|
||||||
try:
|
try:
|
||||||
_materialise_once(session, env)
|
_materialise_once(session, env)
|
||||||
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only)
|
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it
|
||||||
except RunnerError as exc:
|
except RunnerError as exc:
|
||||||
log.error("runner failed: %s", exc)
|
log.error("runner failed: %s", exc)
|
||||||
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
|
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
|
||||||
|
|||||||
@@ -199,22 +199,6 @@ def _model(summary: object) -> str | None:
|
|||||||
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
||||||
|
|
||||||
|
|
||||||
def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]:
|
|
||||||
""":func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present)."""
|
|
||||||
final = base / f"{case}-final" / "agreed.json"
|
|
||||||
return case_features(
|
|
||||||
facts=_json(base / f"{case}-facts" / "facts.json"),
|
|
||||||
passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES},
|
|
||||||
models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES},
|
|
||||||
agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"),
|
|
||||||
alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"),
|
|
||||||
alerts_summary=_json(base / f"{case}-alerts" / "summary.json"),
|
|
||||||
workspace=workspace,
|
|
||||||
facts_summary=_json(base / f"{case}-facts" / "summary.json"),
|
|
||||||
model_roles=_json(base / f"{case}-roles" / "roles.json"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||||
return Path(env[name]) if env.get(name) else None
|
return Path(env[name]) if env.get(name) else None
|
||||||
|
|
||||||
@@ -266,7 +250,7 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
|||||||
return 0 if summary["status"] == "done" else 2
|
return 0 if summary["status"] == "done" else 2
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"]
|
__all__ = ["DROPPED_ON_READ", "case_features", "main", ]
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import json
|
|||||||
from collections.abc import Collection
|
from collections.abc import Collection
|
||||||
from dataclasses import asdict, dataclass, replace
|
from dataclasses import asdict, dataclass, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Protocol
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from .preprocess import FileTarget
|
from .preprocess import FileTarget
|
||||||
|
|
||||||
@@ -28,11 +28,6 @@ class RankingScore:
|
|||||||
static_boosts: list[str]
|
static_boosts: list[str]
|
||||||
|
|
||||||
|
|
||||||
class RankerClient(Protocol):
|
|
||||||
def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
|
|
||||||
def rank_targets(targets: list[FileTarget], model_rankings: object | None = None, model_id: str | None = None) -> list[RankingScore]:
|
def rank_targets(targets: list[FileTarget], model_rankings: object | None = None, model_id: str | None = None) -> list[RankingScore]:
|
||||||
model_by_path = parse_ranker_response(model_rankings, model_id) if model_rankings is not None else {}
|
model_by_path = parse_ranker_response(model_rankings, model_id) if model_rankings is not None else {}
|
||||||
rankings: list[RankingScore] = []
|
rankings: list[RankingScore] = []
|
||||||
@@ -43,32 +38,6 @@ def rank_targets(targets: list[FileTarget], model_rankings: object | None = None
|
|||||||
return sorted(rankings, key=lambda item: (-item.priority, item.path))
|
return sorted(rankings, key=lambda item: (-item.priority, item.path))
|
||||||
|
|
||||||
|
|
||||||
def rank_targets_with_model(
|
|
||||||
targets: list[FileTarget],
|
|
||||||
*,
|
|
||||||
client: RankerClient,
|
|
||||||
model: str,
|
|
||||||
chunk_size: int = 150,
|
|
||||||
timeout_seconds: int = 60,
|
|
||||||
) -> list[RankingScore]:
|
|
||||||
if chunk_size < 1:
|
|
||||||
raise RankingError("chunk_size must be at least 1")
|
|
||||||
model_rankings: list[dict[str, object]] = []
|
|
||||||
for start in range(0, len(targets), chunk_size):
|
|
||||||
chunk = targets[start : start + chunk_size]
|
|
||||||
response = client.complete_json(
|
|
||||||
model=model,
|
|
||||||
messages=[
|
|
||||||
{"role": "system", "content": _ranker_system_prompt()},
|
|
||||||
{"role": "user", "content": json.dumps([_target_prompt_payload(target) for target in chunk], sort_keys=True)},
|
|
||||||
],
|
|
||||||
timeout_seconds=timeout_seconds,
|
|
||||||
)
|
|
||||||
for ranking in parse_ranker_response(response, model).values():
|
|
||||||
model_rankings.append(asdict(ranking))
|
|
||||||
return rank_targets(targets, {"rankings": model_rankings}, model_id=model)
|
|
||||||
|
|
||||||
|
|
||||||
def write_rankings(rankings: list[RankingScore], path: Path) -> None:
|
def write_rankings(rankings: list[RankingScore], path: Path) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
payload = {"rankings": [asdict(ranking) for ranking in rankings]}
|
payload = {"rankings": [asdict(ranking) for ranking in rankings]}
|
||||||
@@ -205,25 +174,6 @@ def _clamp_score(value: int) -> int:
|
|||||||
return min(5, max(1, value))
|
return min(5, max(1, value))
|
||||||
|
|
||||||
|
|
||||||
def _ranker_system_prompt() -> str:
|
|
||||||
return (
|
|
||||||
"Score each file for security review priority. Return JSON only with a rankings array. "
|
|
||||||
"Each item must include path, surface, influence, reachability, rationale, and optional static_boosts. "
|
|
||||||
"Scores must be integers from 1 to 5."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _target_prompt_payload(target: FileTarget) -> dict[str, object]:
|
|
||||||
return {
|
|
||||||
"path": target.path,
|
|
||||||
"language": target.language,
|
|
||||||
"loc": target.loc,
|
|
||||||
"tags": target.tags,
|
|
||||||
"has_fuzz_entry_point": target.has_fuzz_entry_point,
|
|
||||||
"reachability_hints": target.reachability_hints,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _reachability_from_hints(hints: list[dict[str, object]]) -> int:
|
def _reachability_from_hints(hints: list[dict[str, object]]) -> int:
|
||||||
score = 0
|
score = 0
|
||||||
for hint in hints:
|
for hint in hints:
|
||||||
|
|||||||
@@ -7,20 +7,14 @@ call is conservatively charged its reservation on resume, never silently free.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import asdict, dataclass, replace
|
from dataclasses import asdict, dataclass, replace
|
||||||
from importlib import import_module
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Protocol
|
from typing import Any, Protocol
|
||||||
|
|
||||||
from ..plane.budget import BudgetExhausted, SpendBudget
|
from ..plane.budget import BudgetExhausted, SpendBudget
|
||||||
from ..plane.memory import open_store
|
|
||||||
from .board import Board
|
from .board import Board
|
||||||
from .budget import SearchBudget
|
from .budget import SearchBudget
|
||||||
|
|
||||||
@@ -421,79 +415,3 @@ class Coordinator:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _refs(refs: Any) -> bool:
|
def _refs(refs: Any) -> bool:
|
||||||
return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs)
|
return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs)
|
||||||
|
|
||||||
|
|
||||||
class BatchExecutor:
|
|
||||||
"""Adapter to the existing AX/VM transfer and checkpoint protocol.
|
|
||||||
|
|
||||||
Pass AXLane/DockerLane instances and a trusted worker command. No default
|
|
||||||
worker is invented here: reason/explore entrypoints belong to group 5.
|
|
||||||
The coordinator owns retries, so batch performs exactly one attempt.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, *, image: str, command: tuple[str, ...], output: Path, lanes: dict[str, Any]) -> None:
|
|
||||||
batch = import_module("benchmarks.ax.batch")
|
|
||||||
batch.validate_image(image)
|
|
||||||
if not command:
|
|
||||||
raise ValueError("worker command required")
|
|
||||||
self.image, self.command, self.output, self.lanes = image, command, output, lanes
|
|
||||||
|
|
||||||
def __call__(self, task: SearchTask) -> dict[str, Any]:
|
|
||||||
import copy
|
|
||||||
|
|
||||||
batch = import_module("benchmarks.ax.batch")
|
|
||||||
|
|
||||||
if task.lane not in self.lanes:
|
|
||||||
return {"status": "sandbox_failure", "cost_usd": 0}
|
|
||||||
data = json.dumps(asdict(task), sort_keys=True, allow_nan=False).encode()
|
|
||||||
digest = hashlib.sha256(data).hexdigest()
|
|
||||||
identity = task.search_id + "-" + task.id
|
|
||||||
item = batch.Item(
|
|
||||||
identity,
|
|
||||||
{"SPEC_URL": data},
|
|
||||||
"result.dat",
|
|
||||||
self.command,
|
|
||||||
{"SEARCH_STEP": task.step, "SEARCH_ID": task.search_id, "SEARCH_TASK_ID": task.id},
|
|
||||||
digest,
|
|
||||||
)
|
|
||||||
|
|
||||||
def validate(raw: bytes, output: Path) -> None:
|
|
||||||
if len(raw) > 262144:
|
|
||||||
raise ValueError("search result size exceeded")
|
|
||||||
record = json.loads(raw)
|
|
||||||
if not isinstance(record, dict) or record.get("status") not in (
|
|
||||||
"ok",
|
|
||||||
"sandbox_failure",
|
|
||||||
"instrument_failure",
|
|
||||||
"oom",
|
|
||||||
"execution_failure",
|
|
||||||
):
|
|
||||||
raise ValueError("invalid search result")
|
|
||||||
(output / "result.json").write_text(json.dumps(record))
|
|
||||||
|
|
||||||
workload = batch.Workload(self.image, frozenset({"SPEC_URL", "RESULT_URL"}), task.limits["task_wall_seconds"], validate)
|
|
||||||
lane = copy.copy(self.lanes[task.lane])
|
|
||||||
if hasattr(lane, "workload"):
|
|
||||||
lane.workload = workload
|
|
||||||
out = self.output / task.search_id / task.id
|
|
||||||
batch.dispatch([item], workload, out, {task.lane: lane}, lanes=(task.lane,), max_attempts=1)
|
|
||||||
record_path = out / (hashlib.sha256(identity.encode()).hexdigest() + ".json")
|
|
||||||
if not record_path.exists():
|
|
||||||
return {"status": "execution_failure", "cost_usd": 0}
|
|
||||||
record: dict[str, Any] = json.loads(record_path.read_text())
|
|
||||||
return record
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
"""Offline checkpoint inspection, also useful before resuming with an executor."""
|
|
||||||
parser = argparse.ArgumentParser(description="Inspect a search board checkpoint (no model calls)")
|
|
||||||
parser.add_argument("--memory", required=True)
|
|
||||||
parser.add_argument("--search-id", required=True)
|
|
||||||
parser.add_argument("--head", required=True)
|
|
||||||
args = parser.parse_args()
|
|
||||||
board = Board.resume(open_store(args.memory), args.search_id, args.head, coordinator="inspect")
|
|
||||||
print(board.snapshot(), end="")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import zipfile
|
|||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import asdict, dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
@@ -709,20 +709,3 @@ def verify(
|
|||||||
isolation_mode=mode,
|
isolation_mode=mode,
|
||||||
chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)),
|
chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("affected", type=Path)
|
|
||||||
parser.add_argument("safe", type=Path)
|
|
||||||
parser.add_argument("artefact", type=Path)
|
|
||||||
parser.add_argument("family")
|
|
||||||
args = parser.parse_args()
|
|
||||||
result = verify(Side(args.affected), Side(args.safe), args.artefact, args.family)
|
|
||||||
print(json.dumps(asdict(result), indent=2))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
|
||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
import os
|
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Protocol, cast
|
from typing import Any, Protocol, cast
|
||||||
@@ -15,12 +13,11 @@ import yaml
|
|||||||
|
|
||||||
from ..model.endpoint import Prices
|
from ..model.endpoint import Prices
|
||||||
from ..plane.budget import BudgetExhausted, SpendBudget, cost_of
|
from ..plane.budget import BudgetExhausted, SpendBudget, cost_of
|
||||||
from ..provider.openrouter import OpenRouterChatClient, parse_json_content
|
from ..provider.openrouter import parse_json_content
|
||||||
from . import _sandbox
|
from . import _sandbox
|
||||||
from .budget import SearchBudget
|
from .budget import SearchBudget
|
||||||
from .coordinator import SearchTask
|
from .coordinator import SearchTask
|
||||||
from .demo import DEMO_SCHEMA, FAMILY_ORACLES, MAX_DEMO_BYTES, load_demo, validate_demo, validate_oracle
|
from .demo import DEMO_SCHEMA, FAMILY_ORACLES, MAX_DEMO_BYTES, load_demo, validate_demo, validate_oracle
|
||||||
from .executor import ObjectStoreExecutor
|
|
||||||
from .task_storage import exception_reason
|
from .task_storage import exception_reason
|
||||||
|
|
||||||
|
|
||||||
@@ -525,44 +522,3 @@ class Worker:
|
|||||||
"cost_usd": meter.spent,
|
"cost_usd": meter.spent,
|
||||||
"isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
|
"isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--task", required=True, type=Path)
|
|
||||||
parser.add_argument("--repo", required=True, type=Path)
|
|
||||||
parser.add_argument("--demo", required=True, type=Path)
|
|
||||||
parser.add_argument("--model", required=True)
|
|
||||||
parser.add_argument("--input-per-m", required=True, type=float)
|
|
||||||
parser.add_argument("--output-per-m", required=True, type=float)
|
|
||||||
parser.add_argument("--command-put-url")
|
|
||||||
parser.add_argument("--result-get-url")
|
|
||||||
args = parser.parse_args()
|
|
||||||
if bool(args.command_put_url) != bool(args.result_get_url):
|
|
||||||
parser.error("both executor mailbox URLs are required")
|
|
||||||
executor = ObjectStoreExecutor(args.command_put_url, args.result_get_url) if args.command_put_url else None
|
|
||||||
task = SearchTask(**json.loads(args.task.read_text()))
|
|
||||||
if task.step == "explore" and executor is None:
|
|
||||||
parser.error("explore requires a separate repository executor")
|
|
||||||
client = OpenRouterChatClient(
|
|
||||||
api_key=os.environ["OPENROUTER_API_KEY"],
|
|
||||||
base_url=os.environ.get("OPENROUTER_BASE_URL", OpenRouterChatClient.base_url),
|
|
||||||
max_attempts=1,
|
|
||||||
)
|
|
||||||
worker = Worker(
|
|
||||||
client,
|
|
||||||
repo=args.repo,
|
|
||||||
demo=args.demo,
|
|
||||||
model=args.model,
|
|
||||||
prices=Prices(args.input_per_m, args.input_per_m, args.output_per_m),
|
|
||||||
executor=executor,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
print(json.dumps(worker(task)))
|
|
||||||
finally:
|
|
||||||
if executor is not None:
|
|
||||||
executor.submit("stop", {}, timeout_seconds=10)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|||||||
@@ -15,12 +15,8 @@ from .findings import StaticFinding
|
|||||||
from .hunter_tools import (
|
from .hunter_tools import (
|
||||||
PathEscapesRepo,
|
PathEscapesRepo,
|
||||||
clamp_repo_path,
|
clamp_repo_path,
|
||||||
entry_points,
|
|
||||||
find_refs,
|
find_refs,
|
||||||
flows,
|
|
||||||
grep_repo,
|
grep_repo,
|
||||||
obligations,
|
|
||||||
read_definition,
|
|
||||||
read_file,
|
read_file,
|
||||||
)
|
)
|
||||||
from .provider.openrouter import OpenRouterChatClient, OpenRouterError
|
from .provider.openrouter import OpenRouterChatClient, OpenRouterError
|
||||||
@@ -31,7 +27,6 @@ DEFAULT_MAX_STEPS = 4
|
|||||||
CLIENT_ENV = "OPENULTRASAST_HUNTER_CLIENT"
|
CLIENT_ENV = "OPENULTRASAST_HUNTER_CLIENT"
|
||||||
_FINDING_ID_PREFIX = "tool-hunter:"
|
_FINDING_ID_PREFIX = "tool-hunter:"
|
||||||
DEFAULT_TOOL_NAMES = ("read_file", "grep_repo", "find_refs")
|
DEFAULT_TOOL_NAMES = ("read_file", "grep_repo", "find_refs")
|
||||||
_HUNTER_TOOL_NAMES = frozenset(DEFAULT_TOOL_NAMES) | {"read_definition", "entry_points", "flows", "obligations"}
|
|
||||||
_SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"})
|
_SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"})
|
||||||
_SCRIPTED_FLAGS = frozenset({"scripted", "script"})
|
_SCRIPTED_FLAGS = frozenset({"scripted", "script"})
|
||||||
_DUMP_FLAGS = frozenset({"dump", "dump-only"})
|
_DUMP_FLAGS = frozenset({"dump", "dump-only"})
|
||||||
@@ -91,50 +86,6 @@ HUNTER_TOOLS: list[dict[str, object]] = [
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "read_definition",
|
|
||||||
"description": "Resolve one named function to its file, line span and body. Use it instead of guessing what a callee does.",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"symbol": {"type": "string"}, "path": {"type": "string"}, "max_chars": {"type": "integer"}},
|
|
||||||
"required": ["symbol"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "entry_points",
|
|
||||||
"description": "List the handlers this repository exposes, with their access classification. Identifiers and line spans only.",
|
|
||||||
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": []},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "flows",
|
|
||||||
"description": "List adjudicated source-to-sink records inside one function. Identifiers and line numbers only.",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"path": {"type": "string"}, "function": {"type": "string"}},
|
|
||||||
"required": ["path", "function"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "obligations",
|
|
||||||
"description": "List what one function owes and has not discharged: the operation, the missing discharger and why it is owed.",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"path": {"type": "string"}, "function": {"type": "string"}},
|
|
||||||
"required": ["path", "function"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
|
|
||||||
_SYSTEM_PROMPT = (
|
_SYSTEM_PROMPT = (
|
||||||
@@ -330,15 +281,13 @@ def run_tool_hunter(
|
|||||||
system_prompt: str = _SYSTEM_PROMPT,
|
system_prompt: str = _SYSTEM_PROMPT,
|
||||||
user_prompt: str | None = None,
|
user_prompt: str | None = None,
|
||||||
context_files: Sequence[str] = (),
|
context_files: Sequence[str] = (),
|
||||||
tools: Sequence[str] | None = None,
|
|
||||||
max_chars: int = _DEFAULT_MAX_CHARS,
|
max_chars: int = _DEFAULT_MAX_CHARS,
|
||||||
tags: Sequence[str] = ("tool-hunter",),
|
tags: Sequence[str] = ("tool-hunter",),
|
||||||
) -> list[StaticFinding]:
|
) -> list[StaticFinding]:
|
||||||
"""One bounded hunt. Every keyword defaults to today's behaviour; a per-family detector overrides the
|
"""One bounded hunt with a prompt, starting context and finding tags."""
|
||||||
prompt, the context it starts from, the tools it may call and the tags its findings carry."""
|
|
||||||
if max_steps <= 0:
|
if max_steps <= 0:
|
||||||
return []
|
return []
|
||||||
schemas = _tool_schemas(tools)
|
schemas = HUNTER_TOOLS
|
||||||
messages: list[dict[str, object]] = [
|
messages: list[dict[str, object]] = [
|
||||||
{"role": "system", "content": system_prompt},
|
{"role": "system", "content": system_prompt},
|
||||||
{"role": "user", "content": _user_message(root, hotspots, user_prompt, context_files, max_chars)},
|
{"role": "user", "content": _user_message(root, hotspots, user_prompt, context_files, max_chars)},
|
||||||
@@ -350,7 +299,7 @@ def run_tool_hunter(
|
|||||||
if tool_calls:
|
if tool_calls:
|
||||||
messages.append(_assistant_tool_message(response.content, tool_calls, response.reasoning))
|
messages.append(_assistant_tool_message(response.content, tool_calls, response.reasoning))
|
||||||
for call in tool_calls:
|
for call in tool_calls:
|
||||||
if call.name in _HUNTER_TOOL_NAMES:
|
if call.name in DEFAULT_TOOL_NAMES:
|
||||||
used_tools = True
|
used_tools = True
|
||||||
messages.append(_tool_result_message(root, call, max_chars))
|
messages.append(_tool_result_message(root, call, max_chars))
|
||||||
continue
|
continue
|
||||||
@@ -414,12 +363,6 @@ def _looks_like_json(content: str) -> bool:
|
|||||||
return stripped.startswith(("[", "{"))
|
return stripped.startswith(("[", "{"))
|
||||||
|
|
||||||
|
|
||||||
def _tool_schemas(names: Sequence[str] | None) -> list[dict[str, object]]:
|
|
||||||
wanted = tuple(names) if names is not None else DEFAULT_TOOL_NAMES
|
|
||||||
by_name = {str(schema["function"]["name"]): schema for schema in HUNTER_TOOLS} # type: ignore[index]
|
|
||||||
return [by_name[name] for name in wanted if name in by_name]
|
|
||||||
|
|
||||||
|
|
||||||
def _user_message(root: Path, hotspots: Sequence[Hotspot], user_prompt: str | None, context_files: Sequence[str], max_chars: int) -> str:
|
def _user_message(root: Path, hotspots: Sequence[Hotspot], user_prompt: str | None, context_files: Sequence[str], max_chars: int) -> str:
|
||||||
# Context files FIRST, the question LAST. DeepSeek bills a cached prefix at 1/30 of a fresh one, and the cache
|
# Context files FIRST, the question LAST. DeepSeek bills a cached prefix at 1/30 of a fresh one, and the cache
|
||||||
# is by prefix: with the question first, every candidate in the same file broke the prefix at its first
|
# is by prefix: with the question first, every candidate in the same file broke the prefix at its first
|
||||||
@@ -502,30 +445,13 @@ def _dispatch_tool(root: Path, name: str, arguments: Mapping[str, object], defau
|
|||||||
path = _require_str(arguments, "path")
|
path = _require_str(arguments, "path")
|
||||||
max_chars = _optional_int(arguments, "max_chars", default_max_chars)
|
max_chars = _optional_int(arguments, "max_chars", default_max_chars)
|
||||||
return {"path": path, "content": read_file(root, path, max_chars=max_chars)}
|
return {"path": path, "content": read_file(root, path, max_chars=max_chars)}
|
||||||
if name == "read_definition":
|
|
||||||
symbol = _require_str(arguments, "symbol")
|
|
||||||
scope = arguments.get("path")
|
|
||||||
found = read_definition(
|
|
||||||
root,
|
|
||||||
symbol,
|
|
||||||
max_chars=_optional_int(arguments, "max_chars", default_max_chars),
|
|
||||||
path=str(scope) if isinstance(scope, str) and scope else None,
|
|
||||||
)
|
|
||||||
return found if found is not None else {"symbol": symbol, "found": False}
|
|
||||||
if name == "entry_points":
|
|
||||||
scope = arguments.get("path")
|
|
||||||
return entry_points(root, path=str(scope) if isinstance(scope, str) and scope else None)
|
|
||||||
if name == "flows":
|
|
||||||
return flows(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function"))
|
|
||||||
if name == "obligations":
|
|
||||||
return obligations(root, path=_require_str(arguments, "path"), function=_require_str(arguments, "function"))
|
|
||||||
if name == "grep_repo":
|
if name == "grep_repo":
|
||||||
pattern = _require_str(arguments, "pattern")
|
pattern = _require_str(arguments, "pattern")
|
||||||
max_matches = _optional_int(arguments, "max_matches", _DEFAULT_MAX_MATCHES)
|
max_matches = _optional_int(arguments, "max_matches", _DEFAULT_MAX_MATCHES)
|
||||||
return grep_repo(root, pattern, max_matches=max_matches)
|
return grep_repo(root, pattern, max_matches=max_matches)
|
||||||
if name == "find_refs":
|
if name == "find_refs":
|
||||||
symbol = _require_str(arguments, "symbol")
|
symbol = _require_str(arguments, "symbol")
|
||||||
return find_refs(root, symbol, None)
|
return find_refs(root, symbol)
|
||||||
raise ValueError(f"unknown tool: {name}")
|
raise ValueError(f"unknown tool: {name}")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -36,16 +36,6 @@ class VerificationResult:
|
|||||||
context_sources: list[str]
|
context_sources: list[str]
|
||||||
|
|
||||||
|
|
||||||
def validate_evidence_transition(current: EvidenceLevel | str, target: EvidenceLevel | str) -> EvidenceLevel:
|
|
||||||
current_level = EvidenceLevel(current)
|
|
||||||
target_level = EvidenceLevel(target)
|
|
||||||
if _EVIDENCE_ORDER[target_level] < _EVIDENCE_ORDER[current_level]:
|
|
||||||
raise ValueError(f"cannot move evidence level backward: {current_level} -> {target_level}")
|
|
||||||
if _EVIDENCE_ORDER[target_level] - _EVIDENCE_ORDER[current_level] > 1:
|
|
||||||
raise ValueError(f"cannot skip evidence levels: {current_level} -> {target_level}")
|
|
||||||
return target_level
|
|
||||||
|
|
||||||
|
|
||||||
def is_report_verified(evidence_level: EvidenceLevel | str, status: VerificationStatus | str) -> bool:
|
def is_report_verified(evidence_level: EvidenceLevel | str, status: VerificationStatus | str) -> bool:
|
||||||
level = EvidenceLevel(evidence_level)
|
level = EvidenceLevel(evidence_level)
|
||||||
verification_status = VerificationStatus(status)
|
verification_status = VerificationStatus(status)
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ from openultrasast.calibration import (
|
|||||||
apply_ranking_calibration,
|
apply_ranking_calibration,
|
||||||
calibrate_ranking,
|
calibrate_ranking,
|
||||||
calibrate_rankings,
|
calibrate_rankings,
|
||||||
compute_ranking_metrics,
|
|
||||||
learning_from_finding_outcome,
|
learning_from_finding_outcome,
|
||||||
record_false_positive_learning,
|
record_false_positive_learning,
|
||||||
write_false_positive_learnings,
|
write_false_positive_learnings,
|
||||||
@@ -62,34 +61,6 @@ def test_false_positive_learning_records_reason_and_scope(tmp_path: Path) -> Non
|
|||||||
assert "missing_attacker_control" in output.read_text()
|
assert "missing_attacker_control" in output.read_text()
|
||||||
|
|
||||||
|
|
||||||
def test_ranking_metrics_track_false_positive_reasons_by_tier() -> None:
|
|
||||||
metrics = compute_ranking_metrics(
|
|
||||||
[
|
|
||||||
{"tier": "A", "outcome": FindingOutcome.VERIFIED, "time_to_verification_seconds": 10},
|
|
||||||
{
|
|
||||||
"tier": "A",
|
|
||||||
"outcome": FindingOutcome.REJECTED,
|
|
||||||
"reason": FalsePositiveReason.UNREACHABLE_PATH,
|
|
||||||
"time_to_verification_seconds": 20,
|
|
||||||
},
|
|
||||||
{"tier": "B", "outcome": FindingOutcome.REJECTED, "reason": FalsePositiveReason.DUPLICATE},
|
|
||||||
{
|
|
||||||
"tier": "A",
|
|
||||||
"outcome": FindingOutcome.UNVERIFIED,
|
|
||||||
"reason": FalsePositiveReason.UNVERIFIED,
|
|
||||||
"missed_fixture_vulnerabilities": 1,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
tier="A",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert metrics.verified_findings == 1
|
|
||||||
assert metrics.false_positives == 2
|
|
||||||
assert metrics.time_to_verification_seconds == 15.0
|
|
||||||
assert metrics.missed_fixture_vulnerabilities == 1
|
|
||||||
assert metrics.false_positive_reasons["unreachable_path"] == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None:
|
def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None:
|
||||||
learning = record_false_positive_learning(
|
learning = record_false_positive_learning(
|
||||||
_finding("auth/login.py"),
|
_finding("auth/login.py"),
|
||||||
|
|||||||
+1
-15
@@ -1,7 +1,7 @@
|
|||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from openultrasast.findings import build_quick_hunter_prompt, quick_scan_findings, write_findings
|
from openultrasast.findings import quick_scan_findings, write_findings
|
||||||
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
|
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
|
||||||
from openultrasast.preprocess import preprocess_repository
|
from openultrasast.preprocess import preprocess_repository
|
||||||
from openultrasast.rank import rank_targets
|
from openultrasast.rank import rank_targets
|
||||||
@@ -92,17 +92,3 @@ def test_findings_inherit_only_matching_function_level_reachability(tmp_path: Pa
|
|||||||
assert unreachable.ranking_priority <= 1.5
|
assert unreachable.ranking_priority <= 1.5
|
||||||
|
|
||||||
|
|
||||||
def test_quick_hunter_prompt_is_bounded() -> None:
|
|
||||||
repo = Path("/repo/example.py")
|
|
||||||
target = next(iter(preprocess_repository(repo.parent)[1]), None) if repo.parent.exists() else None
|
|
||||||
assert target is None
|
|
||||||
|
|
||||||
class Target:
|
|
||||||
path = "example.py"
|
|
||||||
language = "python"
|
|
||||||
tags = ["auth_boundary"]
|
|
||||||
|
|
||||||
prompt = build_quick_hunter_prompt(Target(), "x" * 5000) # type: ignore[arg-type]
|
|
||||||
|
|
||||||
assert "example.py" in prompt
|
|
||||||
assert len(prompt) < 4300
|
|
||||||
|
|||||||
+1
-15
@@ -6,13 +6,7 @@ from pathlib import Path
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from openultrasast.findings import StaticFinding
|
from openultrasast.findings import StaticFinding
|
||||||
from openultrasast.fusion import (
|
from openultrasast.fusion import FusionDisposition, fuse_finding, fuse_findings, fusion_triggers
|
||||||
FusionDisposition,
|
|
||||||
fuse_finding,
|
|
||||||
fuse_findings,
|
|
||||||
fusion_triggers,
|
|
||||||
should_fuse,
|
|
||||||
)
|
|
||||||
from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus
|
from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus
|
||||||
|
|
||||||
|
|
||||||
@@ -63,10 +57,6 @@ def test_triggers_on_high_severity_and_reachable() -> None:
|
|||||||
assert "gates_risky_fix" in triggers
|
assert "gates_risky_fix" in triggers
|
||||||
|
|
||||||
|
|
||||||
def test_low_severity_unreachable_does_not_trigger() -> None:
|
|
||||||
assert not should_fuse(_finding(severity="low", reachability="unknown"))
|
|
||||||
|
|
||||||
|
|
||||||
def test_verifier_disagreement_triggers() -> None:
|
def test_verifier_disagreement_triggers() -> None:
|
||||||
triggers = fusion_triggers(
|
triggers = fusion_triggers(
|
||||||
_finding(severity="low", reachability="unknown"),
|
_finding(severity="low", reachability="unknown"),
|
||||||
@@ -83,10 +73,6 @@ def test_evidence_conflict_triggers_when_rejected_but_reachable() -> None:
|
|||||||
assert "evidence_conflict" in triggers
|
assert "evidence_conflict" in triggers
|
||||||
|
|
||||||
|
|
||||||
def test_high_assurance_request_forces_a_trigger() -> None:
|
|
||||||
assert should_fuse(_finding(severity="low", reachability="unknown"), high_assurance=True)
|
|
||||||
|
|
||||||
|
|
||||||
# ---- deterministic two-panel decider + dispositions -------------------------
|
# ---- deterministic two-panel decider + dispositions -------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+4
-68
@@ -1,100 +1,36 @@
|
|||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from openultrasast.config import load_config
|
from openultrasast.config import load_config
|
||||||
from openultrasast.harness import (
|
from openultrasast.harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||||
HarnessContractError,
|
|
||||||
HarnessEvent,
|
|
||||||
HarnessRuntime,
|
|
||||||
HarnessTraceWriter,
|
|
||||||
ProcessorSpec,
|
|
||||||
write_harness_config,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
def test_runtime_emits_stage_events(tmp_path: Path) -> None:
|
||||||
class RecordingProcessor:
|
|
||||||
spec: ProcessorSpec
|
|
||||||
|
|
||||||
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
return {"seen_stage": event.stage}
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class BadWriteProcessor:
|
|
||||||
spec: ProcessorSpec
|
|
||||||
|
|
||||||
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
return {"undeclared": event.event_type}
|
|
||||||
|
|
||||||
|
|
||||||
def test_runtime_emits_stage_events_and_processor_state(tmp_path: Path) -> None:
|
|
||||||
trace_path = tmp_path / "events.jsonl"
|
trace_path = tmp_path / "events.jsonl"
|
||||||
processor = RecordingProcessor(ProcessorSpec(name="recorder", version="1", handles=("stage_start",), writes=("seen_stage",)))
|
|
||||||
runtime = HarnessRuntime(
|
runtime = HarnessRuntime(
|
||||||
scan_id="scan-1",
|
scan_id="scan-1",
|
||||||
config=load_config(None),
|
config=load_config(None),
|
||||||
trace_writer=HarnessTraceWriter(trace_path),
|
trace_writer=HarnessTraceWriter(trace_path),
|
||||||
processors=[processor],
|
|
||||||
)
|
)
|
||||||
|
|
||||||
result = runtime.run_stage("preprocess", lambda: "ok")
|
result = runtime.run_stage("preprocess", lambda: "ok")
|
||||||
|
|
||||||
events = [json.loads(line) for line in trace_path.read_text().splitlines()]
|
events = [json.loads(line) for line in trace_path.read_text().splitlines()]
|
||||||
assert result == "ok"
|
assert result == "ok"
|
||||||
assert runtime.state["seen_stage"] == "preprocess"
|
assert [event["event_type"] for event in events] == ["stage_start", "stage_end"]
|
||||||
assert [event["event_type"] for event in events] == ["stage_start", "stage_start", "stage_end"]
|
|
||||||
assert events[1]["payload"] == {"changed_state": ["seen_stage"], "processor": "recorder"}
|
|
||||||
|
|
||||||
|
|
||||||
def test_strict_contract_mode_rejects_undeclared_writes(tmp_path: Path) -> None:
|
def test_harness_config_serializes_models_and_prompt_hashes(tmp_path: Path) -> None:
|
||||||
processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
|
|
||||||
runtime = HarnessRuntime(
|
|
||||||
scan_id="scan-1",
|
|
||||||
config=load_config(None),
|
|
||||||
trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
|
|
||||||
processors=[processor],
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(HarnessContractError, match="undeclared writes"):
|
|
||||||
runtime.emit("stage_start", "rank", {})
|
|
||||||
|
|
||||||
|
|
||||||
def test_warn_contract_mode_records_degradation(tmp_path: Path) -> None:
|
|
||||||
processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
|
|
||||||
runtime = HarnessRuntime(
|
|
||||||
scan_id="scan-1",
|
|
||||||
config=load_config(None),
|
|
||||||
trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
|
|
||||||
processors=[processor],
|
|
||||||
contract_mode="warn",
|
|
||||||
)
|
|
||||||
|
|
||||||
runtime.emit("stage_start", "rank", {})
|
|
||||||
|
|
||||||
assert "undeclared writes" in runtime.state["degradations"][0]
|
|
||||||
assert '"status": "degraded"' in (tmp_path / "events.jsonl").read_text()
|
|
||||||
|
|
||||||
|
|
||||||
def test_harness_config_serializes_models_processors_and_prompt_hashes(tmp_path: Path) -> None:
|
|
||||||
config_path = tmp_path / "openultrasast.toml"
|
config_path = tmp_path / "openultrasast.toml"
|
||||||
config_path.write_text('[models]\nranker = "openrouter/ranker"\n')
|
config_path.write_text('[models]\nranker = "openrouter/ranker"\n')
|
||||||
processor = RecordingProcessor(ProcessorSpec(name="recorder", version="2026-01-01", handles=("stage_start",), writes=("seen_stage",)))
|
|
||||||
output = tmp_path / "harness.json"
|
output = tmp_path / "harness.json"
|
||||||
|
|
||||||
write_harness_config(
|
write_harness_config(
|
||||||
config=load_config(config_path),
|
config=load_config(config_path),
|
||||||
processors=[processor],
|
|
||||||
contract_mode="strict",
|
|
||||||
path=output,
|
path=output,
|
||||||
prompts={"ranker": "score files"},
|
prompts={"ranker": "score files"},
|
||||||
)
|
)
|
||||||
|
|
||||||
payload = json.loads(output.read_text())
|
payload = json.loads(output.read_text())
|
||||||
assert payload["model_roles"]["ranker"] == "openrouter/ranker"
|
assert payload["model_roles"]["ranker"] == "openrouter/ranker"
|
||||||
assert payload["processor_versions"] == {"recorder": "2026-01-01"}
|
|
||||||
assert len(payload["prompt_hashes"]["ranker"]) == 64
|
assert len(payload["prompt_hashes"]["ranker"]) == 64
|
||||||
|
|||||||
@@ -1,198 +0,0 @@
|
|||||||
"""learning-harness task 1.5: curated repository tools and hunter-loop parameters (offline)."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from openultrasast.hunter_tools import PathEscapesRepo
|
|
||||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter
|
|
||||||
|
|
||||||
APP = (
|
|
||||||
"from flask import request\n\n\n"
|
|
||||||
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
|
|
||||||
" return Book.query.filter_by(book_title=title).first()\n\n\n"
|
|
||||||
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
|
|
||||||
" owner = request.args.get('owner')\n"
|
|
||||||
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n"
|
|
||||||
"def helper(value):\n"
|
|
||||||
" return value.strip()\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def repo(tmp_path: Path) -> Path:
|
|
||||||
(tmp_path / "app.py").write_text(APP)
|
|
||||||
return tmp_path
|
|
||||||
|
|
||||||
|
|
||||||
def test_read_definition_returns_one_span_and_refuses_to_leave_the_repository(repo: Path) -> None:
|
|
||||||
from openultrasast.hunter_tools import read_definition
|
|
||||||
|
|
||||||
found = read_definition(repo, "leaky", max_chars=4000)
|
|
||||||
assert found is not None
|
|
||||||
assert found["path"] == "app.py" and found["name"] == "leaky"
|
|
||||||
assert found["start_line"] == 10 and found["end_line"] == 13
|
|
||||||
assert str(found["text"]).startswith("@app.route('/books/any/<title>')") # a handler's decorators come with it
|
|
||||||
assert "def leaky" in str(found["text"]) and "def guarded" not in str(found["text"])
|
|
||||||
assert read_definition(repo, "not_a_function", max_chars=4000) is None
|
|
||||||
with pytest.raises(PathEscapesRepo):
|
|
||||||
read_definition(repo, "leaky", max_chars=4000, path="../outside.py")
|
|
||||||
|
|
||||||
|
|
||||||
def test_entry_points_lists_handlers_with_their_access_and_never_source_text(repo: Path) -> None:
|
|
||||||
from openultrasast.hunter_tools import entry_points
|
|
||||||
|
|
||||||
rows = entry_points(repo)
|
|
||||||
by_function = {str(row["function"]): row for row in rows if row.get("function")}
|
|
||||||
assert {"guarded", "leaky"} <= set(by_function)
|
|
||||||
assert by_function["guarded"]["access_level"] == "authenticated" and by_function["leaky"]["access_level"] == "public"
|
|
||||||
assert by_function["leaky"]["path"] == "app.py" and isinstance(by_function["leaky"]["line"], int)
|
|
||||||
assert all("text" not in row and "content" not in row for row in rows) # identifiers and spans only
|
|
||||||
assert entry_points(repo, path="app.py") == rows
|
|
||||||
with pytest.raises(PathEscapesRepo):
|
|
||||||
entry_points(repo, path="../elsewhere.py")
|
|
||||||
|
|
||||||
|
|
||||||
def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path) -> None:
|
|
||||||
from openultrasast.hunter_tools import obligations
|
|
||||||
|
|
||||||
rows = obligations(repo, path="app.py", function="leaky")
|
|
||||||
assert rows and all(row["function"] == "leaky" for row in rows)
|
|
||||||
assert {"protected_read"} == {str(row["operation"]) for row in rows}
|
|
||||||
assert "identity_constraint" in {str(row["missing"]) for row in rows}
|
|
||||||
assert all("text" not in row and "content" not in row for row in rows)
|
|
||||||
assert obligations(repo, path="app.py", function="helper") == []
|
|
||||||
with pytest.raises(PathEscapesRepo):
|
|
||||||
obligations(repo, path="../outside.py", function="leaky")
|
|
||||||
|
|
||||||
|
|
||||||
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
|
|
||||||
from openultrasast.hunter_tools import flows
|
|
||||||
|
|
||||||
(repo / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n")
|
|
||||||
rows = flows(repo, path="flow.py", function="run")
|
|
||||||
assert all(row["function"] == "run" and row["path"] == "flow.py" for row in rows)
|
|
||||||
assert all(set(row) <= {"path", "function", "line", "sources", "sinks", "disposition", "cwe"} for row in rows)
|
|
||||||
assert flows(repo, path="app.py", function="helper") == []
|
|
||||||
with pytest.raises(PathEscapesRepo):
|
|
||||||
flows(repo, path="/etc/passwd", function="run")
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_loop_keeps_todays_behaviour_by_default(repo: Path) -> None:
|
|
||||||
from openultrasast.complexity.map import Hotspot
|
|
||||||
from openultrasast.tool_hunter import DEFAULT_TOOL_NAMES
|
|
||||||
|
|
||||||
hotspot = Hotspot(
|
|
||||||
path="app.py",
|
|
||||||
function_name=None,
|
|
||||||
score=1.0,
|
|
||||||
band="high",
|
|
||||||
signals={},
|
|
||||||
rationale="fixture",
|
|
||||||
test_hint=None,
|
|
||||||
inventory_finding_ids=(),
|
|
||||||
)
|
|
||||||
client = ScriptedChatClient(
|
|
||||||
[
|
|
||||||
ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)),
|
|
||||||
ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
findings = run_tool_hunter(repo, [hotspot], client=client, model="m", max_steps=4)
|
|
||||||
assert [finding.finding_id for finding in findings] == ["tool-hunter:app.py:12"]
|
|
||||||
assert findings[0].tags == ["tool-hunter"] and findings[0].evidence_level == "suspicion"
|
|
||||||
assert tuple(tool["function"]["name"] for tool in client.calls[0]["tools"]) == DEFAULT_TOOL_NAMES
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_loop_takes_a_prompt_context_tools_and_tags_and_replays_reasoning(repo: Path) -> None:
|
|
||||||
from openultrasast.complexity.map import Hotspot
|
|
||||||
|
|
||||||
hotspot = Hotspot(
|
|
||||||
path="app.py",
|
|
||||||
function_name="leaky",
|
|
||||||
score=1.0,
|
|
||||||
band="high",
|
|
||||||
signals={},
|
|
||||||
rationale="fixture",
|
|
||||||
test_hint=None,
|
|
||||||
inventory_finding_ids=(),
|
|
||||||
)
|
|
||||||
client = ScriptedChatClient(
|
|
||||||
[
|
|
||||||
ChatResponse(
|
|
||||||
content="looking",
|
|
||||||
reasoning="first I check the route",
|
|
||||||
tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),),
|
|
||||||
),
|
|
||||||
ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
findings = run_tool_hunter(
|
|
||||||
repo,
|
|
||||||
[hotspot],
|
|
||||||
client=client,
|
|
||||||
model="m",
|
|
||||||
max_steps=4,
|
|
||||||
system_prompt="You hunt access control only.",
|
|
||||||
user_prompt="Look at leaky.",
|
|
||||||
context_files=("app.py",),
|
|
||||||
tools=("read_definition", "obligations"),
|
|
||||||
tags=("family:access_control", "detector:access_control@1"),
|
|
||||||
)
|
|
||||||
first = client.calls[0]
|
|
||||||
assert first["messages"][0]["content"] == "You hunt access control only."
|
|
||||||
assert "Look at leaky." in str(first["messages"][1]["content"]) and "def leaky" in str(first["messages"][1]["content"])
|
|
||||||
assert [tool["function"]["name"] for tool in first["tools"]] == ["read_definition", "obligations"]
|
|
||||||
replayed = [message for message in client.calls[1]["messages"] if message.get("role") == "assistant"]
|
|
||||||
assert replayed and replayed[0]["reasoning_content"] == "first I check the route"
|
|
||||||
tool_reply = json.loads(str(client.calls[1]["messages"][-1]["content"]))
|
|
||||||
assert tool_reply and tool_reply[0]["missing"] == "identity_constraint"
|
|
||||||
assert findings[0].tags == ["family:access_control", "detector:access_control@1"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_loop_asks_once_more_for_the_answer_instead_of_dropping_prose() -> None:
|
|
||||||
"""Measured live: the model spends its step budget on tools and then answers in prose, and the whole hunt is
|
|
||||||
discarded (learning-harness Req 6.3).
|
|
||||||
|
|
||||||
Four steps, three of them tool calls, and the fourth a paragraph that names the sink, the argument and the
|
|
||||||
line — parsed as nothing. A detector that found the bug and reported zero is indistinguishable from one that
|
|
||||||
found nothing, and every family in round zero would have baselined at zero for this reason alone."""
|
|
||||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter
|
|
||||||
|
|
||||||
prose = "The function passes `command` straight to os.system at line 2, an OS command sink."
|
|
||||||
answer = json.dumps([{"path": "app.py", "line": 2, "title": "command injection", "rationale": "os.system(command)"}])
|
|
||||||
client = ScriptedChatClient(
|
|
||||||
[
|
|
||||||
ChatResponse(tool_calls=(ToolCall(id="c1", name="grep_repo", arguments={"pattern": "system"}),)),
|
|
||||||
ChatResponse(tool_calls=(ToolCall(id="c2", name="read_file", arguments={"path": "app.py"}),)),
|
|
||||||
ChatResponse(content=prose),
|
|
||||||
ChatResponse(content=answer), # the one extra turn, asked for explicitly
|
|
||||||
]
|
|
||||||
)
|
|
||||||
with tempfile.TemporaryDirectory() as scratch:
|
|
||||||
root = Path(scratch)
|
|
||||||
(root / "app.py").write_text("import os\ndef run(command):\n return os.system(command)\n")
|
|
||||||
findings = run_tool_hunter(root, [], client=client, model="m", max_steps=3)
|
|
||||||
assert [item.line for item in findings] == [2]
|
|
||||||
final = client.calls[-1]
|
|
||||||
# Asked for JSON in the prompt, not through the provider's JSON mode: with tools attached, DeepSeek's JSON mode
|
|
||||||
# returned `{"type": "json_object"}` and lost the answers (tests/test_tool_hunter.py pins the shapes).
|
|
||||||
assert not final.get("json_object") and final.get("tools") == [], "the last turn is plain text without tools"
|
|
||||||
text = "\n".join(str(message.get("content") or "") for message in final["messages"])
|
|
||||||
assert "JSON" in text and prose in text, "the prose it already wrote is what the final turn is asked to format"
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_hunt_that_never_called_a_tool_still_reports_nothing() -> None:
|
|
||||||
"""The extra turn is for formatting an answer, never for inventing one."""
|
|
||||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, run_tool_hunter
|
|
||||||
|
|
||||||
client = ScriptedChatClient([ChatResponse(content="I think there is an injection here."), ChatResponse(content="[]")])
|
|
||||||
with tempfile.TemporaryDirectory() as scratch:
|
|
||||||
root = Path(scratch)
|
|
||||||
(root / "app.py").write_text("x = 1\n")
|
|
||||||
assert run_tool_hunter(root, [], client=client, model="m", max_steps=3) == []
|
|
||||||
assert len(client.calls) == 1, "no tool was called, so there is nothing to format"
|
|
||||||
@@ -3,7 +3,6 @@ from pathlib import Path
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from openultrasast.hunter_tools import PathEscapesRepo, clamp_repo_path, find_refs, grep_repo, read_file
|
from openultrasast.hunter_tools import PathEscapesRepo, clamp_repo_path, find_refs, grep_repo, read_file
|
||||||
from openultrasast.mapping import EntryPointRecord
|
|
||||||
|
|
||||||
SPLIT_SINK = """\
|
SPLIT_SINK = """\
|
||||||
term = request.args["q"]
|
term = request.args["q"]
|
||||||
@@ -85,42 +84,13 @@ def test_read_file_truncates_and_rejects_symlink_escape(tmp_path: Path) -> None:
|
|||||||
read_file(root, "link.py", max_chars=100)
|
read_file(root, "link.py", max_chars=100)
|
||||||
|
|
||||||
|
|
||||||
def test_find_refs_uses_mapping_index_and_clamped_text_search(tmp_path: Path) -> None:
|
def test_find_refs_searches_source_text(tmp_path: Path) -> None:
|
||||||
root = _repo_with_split_sink(tmp_path)
|
root = _repo_with_split_sink(tmp_path)
|
||||||
mapping_index = [
|
refs = find_refs(root, "query")
|
||||||
{"path": "app.py", "name": "query", "line": 2},
|
assert [ref["line"] for ref in refs] == [2, 3]
|
||||||
EntryPointRecord(
|
assert all(ref["path"] == "app.py" and ref["name"] == "query" and ref["source"] == "text" for ref in refs)
|
||||||
path="app.py",
|
assert find_refs(root, "search") == []
|
||||||
line=3,
|
assert find_refs(root, "") == []
|
||||||
end_line=3,
|
|
||||||
function_name="search",
|
|
||||||
name="search",
|
|
||||||
kind="route",
|
|
||||||
access_level="public",
|
|
||||||
trust_boundary="http_request",
|
|
||||||
access_evidence=[],
|
|
||||||
conditions=[],
|
|
||||||
provenance="test",
|
|
||||||
rationale="fixture",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
query_refs = find_refs(root, "query", mapping_index)
|
|
||||||
assert any(ref["path"] == "app.py" and ref["name"] == "query" for ref in query_refs)
|
|
||||||
assert any(ref.get("source") == "mapping" for ref in query_refs)
|
|
||||||
assert any(ref.get("source") == "text" and "query" in str(ref.get("text", ref.get("name", ""))) for ref in query_refs)
|
|
||||||
|
|
||||||
search_refs = find_refs(root, "search", mapping_index)
|
|
||||||
assert any(ref["path"] == "app.py" and ref["name"] == "search" for ref in search_refs)
|
|
||||||
|
|
||||||
|
|
||||||
def test_find_refs_rejects_mapping_path_escape(tmp_path: Path) -> None:
|
|
||||||
root = _repo_with_split_sink(tmp_path)
|
|
||||||
|
|
||||||
with pytest.raises(PathEscapesRepo):
|
|
||||||
find_refs(root, "passwd", [{"path": "../etc/passwd", "name": "passwd"}])
|
|
||||||
with pytest.raises(PathEscapesRepo):
|
|
||||||
find_refs(root, "passwd", [{"path": "/etc/passwd", "name": "passwd"}])
|
|
||||||
|
|
||||||
|
|
||||||
def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> None:
|
def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> None:
|
||||||
|
|||||||
@@ -5,17 +5,7 @@ from __future__ import annotations
|
|||||||
import pytest
|
import pytest
|
||||||
from learn_fixtures import corpus
|
from learn_fixtures import corpus
|
||||||
|
|
||||||
from openultrasast.learn.folds import (
|
from openultrasast.learn.folds import Fold, FoldError, assert_disjoint, compile_split, folds_digest, outer_folds
|
||||||
INSUFFICIENT,
|
|
||||||
Fold,
|
|
||||||
FoldError,
|
|
||||||
assert_disjoint,
|
|
||||||
compile_split,
|
|
||||||
folds_digest,
|
|
||||||
leave_one_framework_out,
|
|
||||||
leave_one_source_out,
|
|
||||||
outer_folds,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None:
|
def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None:
|
||||||
@@ -26,8 +16,6 @@ def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation
|
|||||||
assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4
|
assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4
|
||||||
folds = [
|
folds = [
|
||||||
*outer_folds(examples, seed=1, exclude=split.groups),
|
*outer_folds(examples, seed=1, exclude=split.groups),
|
||||||
*leave_one_source_out(examples, exclude=split.groups),
|
|
||||||
*leave_one_framework_out(examples, exclude=split.groups, min_groups=2),
|
|
||||||
]
|
]
|
||||||
assert_disjoint(split, folds)
|
assert_disjoint(split, folds)
|
||||||
for fold in folds:
|
for fold in folds:
|
||||||
@@ -52,19 +40,6 @@ def test_folds_are_deterministic_by_seed() -> None:
|
|||||||
assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6))
|
assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6))
|
||||||
|
|
||||||
|
|
||||||
def test_leave_one_source_and_framework_out() -> None:
|
|
||||||
examples = corpus(groups=12)
|
|
||||||
sources = leave_one_source_out(examples)
|
|
||||||
assert {f.held_source for f in sources} == {"pairs", "population-v1", "population-v2"}
|
|
||||||
for fold in sources:
|
|
||||||
assert fold.eval_groups == {e.group for e in examples if e.source == fold.held_source}
|
|
||||||
frameworks = {f.held_framework: f for f in leave_one_framework_out(examples)}
|
|
||||||
assert set(frameworks) == {"flask", "django"}
|
|
||||||
assert all(f.status == INSUFFICIENT and not f.eval_groups for f in frameworks.values()) # 3 groups each < 10
|
|
||||||
enough = {f.held_framework: f for f in leave_one_framework_out(examples, min_groups=3)}
|
|
||||||
assert enough["flask"].status == "ok" and enough["flask"].eval_groups == {e.group for e in examples if "flask" in e.frameworks}
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_shared_group_is_refused() -> None:
|
def test_a_shared_group_is_refused() -> None:
|
||||||
examples = corpus(groups=8)
|
examples = corpus(groups=8)
|
||||||
split = compile_split(examples, seed=0)
|
split = compile_split(examples, seed=0)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from dataclasses import replace
|
|||||||
import pytest
|
import pytest
|
||||||
from learn_fixtures import by_id, corpus, example, x_record
|
from learn_fixtures import by_id, corpus, example, x_record
|
||||||
|
|
||||||
from openultrasast.learn.folds import Fold, compile_split, leave_one_framework_out, leave_one_source_out, outer_folds
|
from openultrasast.learn.folds import Fold, compile_split, outer_folds
|
||||||
from openultrasast.learn.retrieve import (
|
from openultrasast.learn.retrieve import (
|
||||||
BoundaryViolation,
|
BoundaryViolation,
|
||||||
Target,
|
Target,
|
||||||
@@ -166,13 +166,11 @@ def _all_folds(examples: list) -> list:
|
|||||||
split = compile_split(examples, seed=3)
|
split = compile_split(examples, seed=3)
|
||||||
return [
|
return [
|
||||||
*outer_folds(examples, seed=3, exclude=split.groups),
|
*outer_folds(examples, seed=3, exclude=split.groups),
|
||||||
*leave_one_source_out(examples, exclude=split.groups),
|
|
||||||
*[f for f in leave_one_framework_out(examples, exclude=split.groups, min_groups=2) if f.status == "ok"],
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_boundary() -> None:
|
def test_boundary() -> None:
|
||||||
"""For outer, leave-one-source-out and leave-one-framework-out folds, every retrieved example and every
|
"""For outer folds, every retrieved example and every
|
||||||
demonstration that reaches a prompt is eligible; ``seed_memory`` writes no example of the case's group."""
|
demonstration that reaches a prompt is eligible; ``seed_memory`` writes no example of the case's group."""
|
||||||
examples = corpus(groups=16)
|
examples = corpus(groups=16)
|
||||||
index = by_id(examples)
|
index = by_id(examples)
|
||||||
@@ -186,7 +184,7 @@ def test_boundary() -> None:
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
folds = _all_folds(examples)
|
folds = _all_folds(examples)
|
||||||
assert {f.kind for f in folds} == {"outer", "leave_one_source_out", "leave_one_framework_out"}
|
assert {f.kind for f in folds} == {"outer"}
|
||||||
checked = 0
|
checked = 0
|
||||||
for fold in folds:
|
for fold in folds:
|
||||||
for target_example in [e for e in examples if e.group in fold.eval_groups]:
|
for target_example in [e for e in examples if e.group in fold.eval_groups]:
|
||||||
@@ -206,19 +204,6 @@ def test_boundary() -> None:
|
|||||||
assert all(e.group != group for e in seed_memory(examples, group))
|
assert all(e.group != group for e in seed_memory(examples, group))
|
||||||
|
|
||||||
|
|
||||||
def test_a_demonstration_of_the_held_out_framework_is_swapped_for_its_alternate() -> None:
|
|
||||||
flask = example("demo-flask", group="d/flask", label=1, frameworks=("flask",))
|
|
||||||
alt = example("demo-plain", group="d/plain", label=1)
|
|
||||||
target_example = example("t", group="t/t", label=0, frameworks=("flask",))
|
|
||||||
target = Target.of(target_example)
|
|
||||||
fold = leave_one_framework_out([flask, target_example], min_groups=1)[0]
|
|
||||||
assert fold.held_framework == "flask" and target_example.group in fold.eval_groups
|
|
||||||
demos = [{"example_id": flask.id, "alternate": {"example_id": alt.id}}]
|
|
||||||
usable = demonstrations_for(demos, by_id([flask, alt]), target, fold)
|
|
||||||
assert [d["example_id"] for d in usable] == [alt.id]
|
|
||||||
assert demonstrations_for([{"example_id": flask.id, "alternate": None}], by_id([flask]), target, fold) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_assert_boundary_catches_an_ineligible_rendered_id() -> None:
|
def test_assert_boundary_catches_an_ineligible_rendered_id() -> None:
|
||||||
examples = corpus(groups=6)
|
examples = corpus(groups=6)
|
||||||
fold = outer_folds(examples, k=3)[0]
|
fold = outer_folds(examples, k=3)[0]
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from openultrasast.learn.features import Vocabulary, build_for_scan, function_of, model_sinks_part, roles_part
|
from openultrasast.learn.features import Vocabulary, function_of, model_sinks_part, roles_part
|
||||||
from openultrasast.learn.roles import (
|
from openultrasast.learn.roles import (
|
||||||
RoleSet,
|
RoleSet,
|
||||||
base_roles,
|
base_roles,
|
||||||
@@ -168,22 +168,6 @@ def test_a_language_without_declarations_is_none_not_failed() -> None:
|
|||||||
assert source_part(["def g():", " pass"], "python", "missing").state == "failed"
|
assert source_part(["def g():", " pass"], "python", "missing").state == "failed"
|
||||||
|
|
||||||
|
|
||||||
def test_build_for_scan_records_the_roles_instrument(tmp_path: Path) -> None:
|
|
||||||
from openultrasast.findings import quick_scan_findings
|
|
||||||
from openultrasast.preprocess import preprocess_repository
|
|
||||||
from openultrasast.rank import rank_targets
|
|
||||||
|
|
||||||
_write(tmp_path, "app/one.py", "import os\n\n\ndef inner(value):\n os.system(value)\n")
|
|
||||||
_, targets = preprocess_repository(tmp_path)
|
|
||||||
assert targets and (tmp_path / targets[0].path).stat().st_size > 0, "the fixture was read"
|
|
||||||
findings = quick_scan_findings(tmp_path, targets, rank_targets(targets))
|
|
||||||
records = build_for_scan(tmp_path, findings, None)
|
|
||||||
assert records and records[0]["instruments"]["roles"]["state"] == "ran"
|
|
||||||
assert records[0]["x"]["roles.sink_confidence"] == 1.0
|
|
||||||
off = build_for_scan(tmp_path, findings, None, roles=False)
|
|
||||||
assert off[0]["instruments"]["roles"]["state"] == "none" and off[0]["x"]["roles.sink_confidence"] is None
|
|
||||||
|
|
||||||
|
|
||||||
# --- model roles: the plane task under a scripted client ----------------------------------------------------------
|
# --- model roles: the plane task under a scripted client ----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,14 +10,10 @@ from typing import Any
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from openultrasast.findings import quick_scan_findings
|
|
||||||
from openultrasast.learn import schema
|
from openultrasast.learn import schema
|
||||||
from openultrasast.learn.features import Delta, Part, build_for_scan, record
|
from openultrasast.learn.features import Part, record
|
||||||
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
|
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
|
||||||
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
|
|
||||||
from openultrasast.plane.tasks.features import case_features
|
from openultrasast.plane.tasks.features import case_features
|
||||||
from openultrasast.preprocess import preprocess_repository
|
|
||||||
from openultrasast.rank import rank_targets
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
APP = """import os
|
APP = """import os
|
||||||
@@ -32,22 +28,6 @@ def {outer}():
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def _scan(root: Path) -> list[dict[str, Any]]:
|
|
||||||
_, targets = preprocess_repository(root)
|
|
||||||
assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read"
|
|
||||||
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
|
|
||||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
|
||||||
assert findings, "the fixture's os.system fires a quick rule"
|
|
||||||
engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []}
|
|
||||||
return build_for_scan(root, findings, engine)
|
|
||||||
|
|
||||||
|
|
||||||
def _repo(root: Path, path: str, inner: str, outer: str) -> Path:
|
|
||||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
(root / path).write_text(APP.format(inner=inner, outer=outer))
|
|
||||||
return root
|
|
||||||
|
|
||||||
|
|
||||||
# --- the allow-list ----------------------------------------------------------------------------------------------
|
# --- the allow-list ----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -63,35 +43,10 @@ def test_features_are_allow_listed(tmp_path: Path) -> None:
|
|||||||
mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
|
mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
|
||||||
assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
|
assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
|
||||||
assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
|
assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
|
||||||
records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta"))
|
|
||||||
assert records
|
|
||||||
for rec in records:
|
|
||||||
assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES"
|
|
||||||
assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"])
|
|
||||||
assert {s.name for s in schema.features_for("static")} < set(names)
|
assert {s.name for s in schema.features_for("static")} < set(names)
|
||||||
assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]
|
assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]
|
||||||
|
|
||||||
|
|
||||||
def test_record_is_name_invariant(tmp_path: Path) -> None:
|
|
||||||
"""Two repositories that differ only in repository name, paths, file and function names and commit ids give the
|
|
||||||
same features: nothing identifies a repository."""
|
|
||||||
first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta"))
|
|
||||||
second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta"))
|
|
||||||
assert [r["x"] for r in first] == [r["x"] for r in second]
|
|
||||||
assert [r["instruments"] for r in first] == [r["instruments"] for r in second]
|
|
||||||
assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope"
|
|
||||||
assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None
|
|
||||||
root = tmp_path / "acme-shop"
|
|
||||||
_, targets = preprocess_repository(root)
|
|
||||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
|
||||||
changed = {"app/one.py": [(5, 5)]}
|
|
||||||
one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40))
|
|
||||||
two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40))
|
|
||||||
assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40
|
|
||||||
assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1
|
|
||||||
assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature"
|
|
||||||
|
|
||||||
|
|
||||||
# --- label fields ------------------------------------------------------------------------------------------------
|
# --- label fields ------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
AGREED = {
|
AGREED = {
|
||||||
@@ -165,17 +120,6 @@ def test_missing_is_not_zero() -> None:
|
|||||||
assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"
|
assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"
|
||||||
|
|
||||||
|
|
||||||
def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None:
|
|
||||||
root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta")
|
|
||||||
_, targets = preprocess_repository(root)
|
|
||||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
|
||||||
unread = build_for_scan(root, findings, {"questions": 0, "findings": []})
|
|
||||||
absent = build_for_scan(root, findings, None)
|
|
||||||
assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"}
|
|
||||||
assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"}
|
|
||||||
assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent))
|
|
||||||
|
|
||||||
|
|
||||||
def test_validate_record_refuses_values_outside_their_type() -> None:
|
def test_validate_record_refuses_values_outside_their_type() -> None:
|
||||||
good = _plane()[0]
|
good = _plane()[0]
|
||||||
validate_record(good)
|
validate_record(good)
|
||||||
|
|||||||
@@ -4,13 +4,7 @@ from pathlib import Path
|
|||||||
from openultrasast.mapping import (
|
from openultrasast.mapping import (
|
||||||
analyze_entry_points,
|
analyze_entry_points,
|
||||||
attach_reachability_hints,
|
attach_reachability_hints,
|
||||||
attach_static_hints,
|
|
||||||
codeql_mapping_tasks,
|
|
||||||
differential_mapping_record,
|
|
||||||
ingest_sarif,
|
ingest_sarif,
|
||||||
semgrep_mapping_tasks,
|
|
||||||
sharp_edge_record,
|
|
||||||
verifier_evidence_candidates,
|
|
||||||
write_entry_points,
|
write_entry_points,
|
||||||
write_static_hints,
|
write_static_hints,
|
||||||
)
|
)
|
||||||
@@ -32,56 +26,6 @@ def test_ingest_sarif_normalizes_semgrep_hints(tmp_path: Path) -> None:
|
|||||||
assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval"
|
assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval"
|
||||||
|
|
||||||
|
|
||||||
def test_static_hints_attach_to_file_targets_and_evidence_candidates(tmp_path: Path) -> None:
|
|
||||||
repo = tmp_path / "repo"
|
|
||||||
repo.mkdir()
|
|
||||||
(repo / "app.py").write_text("eval(user_input)\n")
|
|
||||||
hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "py/code-injection", "app.py"))
|
|
||||||
_, targets = preprocess_repository(repo, static_hints=hints)
|
|
||||||
|
|
||||||
enriched = attach_static_hints(targets, hints)
|
|
||||||
evidence = verifier_evidence_candidates(hints, path="app.py")
|
|
||||||
|
|
||||||
assert enriched[0].static_hints[0]["analyzer"] == "codeql"
|
|
||||||
assert evidence == [
|
|
||||||
{
|
|
||||||
"source": "codeql",
|
|
||||||
"rule_id": "py/code-injection",
|
|
||||||
"path": "app.py",
|
|
||||||
"line": 7,
|
|
||||||
"severity": "high",
|
|
||||||
"message": "Potential injection",
|
|
||||||
"provenance": "sarif:codeql:py/code-injection",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_mapping_task_records_cover_disciplines(tmp_path: Path) -> None:
|
|
||||||
semgrep_hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "parser.c"))
|
|
||||||
codeql_hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "cpp/overflow", "parser.c"))
|
|
||||||
|
|
||||||
assert semgrep_mapping_tasks(semgrep_hints)[0].task == "pattern_variant_review"
|
|
||||||
assert codeql_mapping_tasks(codeql_hints)[0].task == "source_sink_sanitizer_path_review"
|
|
||||||
assert (
|
|
||||||
differential_mapping_record(
|
|
||||||
"parser.c",
|
|
||||||
change_kind="modified_function",
|
|
||||||
trust_boundary="network_input",
|
|
||||||
rationale="Parser changed behind a socket boundary.",
|
|
||||||
).trust_boundary
|
|
||||||
== "network_input"
|
|
||||||
)
|
|
||||||
assert (
|
|
||||||
sharp_edge_record(
|
|
||||||
"config.py",
|
|
||||||
category="insecure_default",
|
|
||||||
surface="debug mode",
|
|
||||||
rationale="Debug default changes production risk.",
|
|
||||||
).category
|
|
||||||
== "insecure_default"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_write_static_hints_artifact(tmp_path: Path) -> None:
|
def test_write_static_hints_artifact(tmp_path: Path) -> None:
|
||||||
hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py"))
|
hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py"))
|
||||||
output = tmp_path / "mapping" / "static_hints.json"
|
output = tmp_path / "mapping" / "static_hints.json"
|
||||||
|
|||||||
@@ -26,17 +26,17 @@ def _cpg(rows): # type: ignore[no-untyped-def]
|
|||||||
|
|
||||||
|
|
||||||
def test_a_permissive_literal_is_entailed() -> None:
|
def test_a_permissive_literal_is_entailed() -> None:
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
|
|
||||||
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}]
|
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="create_app")
|
answer = verdicts(_cpg(rows), _spec(), function="create_app")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
assert "*" in answer.witness and "CORS" in answer.witness
|
assert "*" in answer.witness and "CORS" in answer.witness
|
||||||
|
|
||||||
|
|
||||||
def test_a_restrictive_literal_yields_no_finding() -> None:
|
def test_a_restrictive_literal_yields_no_finding() -> None:
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -47,12 +47,12 @@ def test_a_restrictive_literal_yields_no_finding() -> None:
|
|||||||
"args": ["app", '"https://app.example"'],
|
"args": ["app", '"https://app.example"'],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
assert verdict(_cpg(rows), _spec(), function="create_app") is None
|
assert verdicts(_cpg(rows), _spec(), function="create_app") == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
|
def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
|
||||||
"""The value is computed, so constant abstraction cannot decide it — that residual is the LLM's."""
|
"""The value is computed, so constant abstraction cannot decide it — that residual is the LLM's."""
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
@@ -64,30 +64,30 @@ def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
|
|||||||
"args": ["app", "configured_origins"],
|
"args": ["app", "configured_origins"],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="create_app")
|
answer = verdicts(_cpg(rows), _spec(), function="create_app")[0]
|
||||||
assert answer is not None and answer.rung is Rung.CORROBORATED
|
assert answer is not None and answer.rung is Rung.CORROBORATED
|
||||||
|
|
||||||
|
|
||||||
def test_a_setting_that_is_not_in_the_spec_is_ignored() -> None:
|
def test_a_setting_that_is_not_in_the_spec_is_ignored() -> None:
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
|
|
||||||
rows = [{"setting": 'print("*")', "line": "1", "method": "create_app", "literalArgs": ['"*"'], "args": ['"*"']}]
|
rows = [{"setting": 'print("*")', "line": "1", "method": "create_app", "literalArgs": ['"*"'], "args": ['"*"']}]
|
||||||
assert verdict(_cpg(rows), _spec(), function="create_app") is None
|
assert verdicts(_cpg(rows), _spec(), function="create_app") == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None:
|
def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None:
|
||||||
from openultrasast.cpg.backend import CpgResult
|
from openultrasast.cpg.backend import CpgResult
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
|
|
||||||
assert verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") is None
|
assert verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") == []
|
||||||
|
|
||||||
|
|
||||||
def test_the_verdict_is_deterministic() -> None:
|
def test_the_verdict_is_deterministic() -> None:
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
|
|
||||||
rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}]
|
rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}]
|
||||||
cpg = _cpg(rows)
|
cpg = _cpg(rows)
|
||||||
assert verdict(cpg, _spec(), function="login") == verdict(cpg, _spec(), function="login")
|
assert verdicts(cpg, _spec(), function="login")[0] == verdicts(cpg, _spec(), function="login")[0]
|
||||||
|
|
||||||
|
|
||||||
# --- framework coverage and the weak-algorithm arm --------------------------------------------------------
|
# --- framework coverage and the weak-algorithm arm --------------------------------------------------------
|
||||||
@@ -113,24 +113,24 @@ def test_the_settings_table_covers_the_frameworks_in_the_corpus() -> None:
|
|||||||
|
|
||||||
def test_a_weak_algorithm_is_entailed_by_its_literal() -> None:
|
def test_a_weak_algorithm_is_entailed_by_its_literal() -> None:
|
||||||
"""CWE-327: the danger is the algorithm NAMED, not a permissive flag."""
|
"""CWE-327: the danger is the algorithm NAMED, not a permissive flag."""
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.specs import config_specs
|
from openultrasast.model.specs import config_specs
|
||||||
|
|
||||||
spec = config_specs(language="python")["config_secrets"]
|
spec = config_specs(language="python")["config_secrets"]
|
||||||
rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}]
|
rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}]
|
||||||
answer = verdict(_cpg(rows), spec, function="digest")
|
answer = verdicts(_cpg(rows), spec, function="digest")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
assert "md5" in answer.witness.lower()
|
assert "md5" in answer.witness.lower()
|
||||||
|
|
||||||
|
|
||||||
def test_a_strong_algorithm_yields_no_finding() -> None:
|
def test_a_strong_algorithm_yields_no_finding() -> None:
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
from openultrasast.model.specs import config_specs
|
from openultrasast.model.specs import config_specs
|
||||||
|
|
||||||
spec = config_specs(language="python")["config_secrets"]
|
spec = config_specs(language="python")["config_secrets"]
|
||||||
rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}]
|
rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}]
|
||||||
assert verdict(_cpg(rows), spec, function="digest") is None
|
assert verdicts(_cpg(rows), spec, function="digest") == []
|
||||||
|
|
||||||
|
|
||||||
def _javascript_spec(): # type: ignore[no-untyped-def]
|
def _javascript_spec(): # type: ignore[no-untyped-def]
|
||||||
@@ -142,7 +142,7 @@ def _javascript_spec(): # type: ignore[no-untyped-def]
|
|||||||
|
|
||||||
def test_disabling_template_escaping_is_entailed() -> None:
|
def test_disabling_template_escaping_is_entailed() -> None:
|
||||||
"""The setting NodeGoat's cross-site scripting turns off, claimed where the evidence is: the value."""
|
"""The setting NodeGoat's cross-site scripting turns off, claimed where the evidence is: the value."""
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
@@ -154,7 +154,7 @@ def test_disabling_template_escaping_is_entailed() -> None:
|
|||||||
"args": ["{ autoescape: false }"],
|
"args": ["{ autoescape: false }"],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _javascript_spec(), function="<module>")
|
answer = verdicts(_cpg(rows), _javascript_spec(), function="<module>")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
assert "swig.setDefaults" in answer.witness and "false" in answer.witness
|
assert "swig.setDefaults" in answer.witness and "false" in answer.witness
|
||||||
|
|
||||||
@@ -167,7 +167,7 @@ def test_enabling_template_escaping_is_not_a_finding() -> None:
|
|||||||
reported on exactly the code that repairs it -- the commented-out `autoescape: true` two lines below
|
reported on exactly the code that repairs it -- the commented-out `autoescape: true` two lines below
|
||||||
NodeGoat's defect.
|
NodeGoat's defect.
|
||||||
"""
|
"""
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -178,12 +178,12 @@ def test_enabling_template_escaping_is_not_a_finding() -> None:
|
|||||||
"args": ["{ autoescape: true }"],
|
"args": ["{ autoescape: true }"],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
assert verdict(_cpg(rows), _javascript_spec(), function="<module>") is None
|
assert verdicts(_cpg(rows), _javascript_spec(), function="<module>") == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
|
def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
|
||||||
"""The control for the other direction: scoping must not quiet the settings it was not about."""
|
"""The control for the other direction: scoping must not quiet the settings it was not about."""
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
@@ -195,7 +195,7 @@ def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
|
|||||||
"args": ["{ origin: true }"],
|
"args": ["{ origin: true }"],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _javascript_spec(), function="createApp")
|
answer = verdicts(_cpg(rows), _javascript_spec(), function="createApp")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
|
|
||||||
|
|
||||||
@@ -206,13 +206,13 @@ def test_the_first_setting_in_a_file_is_the_one_reported() -> None:
|
|||||||
spelling of its line numbers. NodeGoat has a permissive session at line 78 and template escaping disabled
|
spelling of its line numbers. NodeGoat has a permissive session at line 78 and template escaping disabled
|
||||||
at line 135, and which one surfaced flipped on that comparison.
|
at line 135, and which one surfaced flipped on that comparison.
|
||||||
"""
|
"""
|
||||||
from openultrasast.model.config_value import verdict
|
from openultrasast.model.config_value import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{"setting": "swig.setDefaults({ autoescape: false })", "line": "135", "method": "m", "literalArgs": ["false"], "args": []},
|
{"setting": "swig.setDefaults({ autoescape: false })", "line": "135", "method": "m", "literalArgs": ["false"], "args": []},
|
||||||
{"setting": "cors({ origin: true })", "line": "78", "method": "m", "literalArgs": ["true"], "args": []},
|
{"setting": "cors({ origin: true })", "line": "78", "method": "m", "literalArgs": ["true"], "args": []},
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _javascript_spec(), function="m")
|
answer = verdicts(_cpg(rows), _javascript_spec(), function="m")[0]
|
||||||
assert answer is not None and "line 78" in answer.witness, answer.witness
|
assert answer is not None and "line 78" in answer.witness, answer.witness
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1).
|
|
||||||
|
|
||||||
Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a
|
|
||||||
real error in this project's history:
|
|
||||||
|
|
||||||
per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the
|
|
||||||
hand-written development slice scored 83%
|
|
||||||
the overfitting gap published beside the headline, so the distance between the development slice and the
|
|
||||||
real-world one is never something a reader has to compute
|
|
||||||
the rung on every a finding's strength is what established it; a report that omits the rung invites the
|
|
||||||
finding reader to treat a suspicion as a result
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
|
|
||||||
def _slice(name, pairs, covered): # type: ignore[no-untyped-def]
|
|
||||||
from openultrasast.model.report import SliceRow
|
|
||||||
|
|
||||||
return SliceRow(slice=name, pairs=pairs, covered=covered)
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_report_carries_a_row_per_slice() -> None:
|
|
||||||
from openultrasast.model.report import render
|
|
||||||
|
|
||||||
text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)])
|
|
||||||
assert "vibe-py" in text and "vfc-js" in text
|
|
||||||
assert "30" in text and "17" in text
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_bare_aggregate_is_refused() -> None:
|
|
||||||
"""Req 11.1: no cross-slice number without the rows it was computed from."""
|
|
||||||
from openultrasast.model.report import render
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="per-slice"):
|
|
||||||
render([])
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_overfitting_gap_is_published_beside_the_headline() -> None:
|
|
||||||
"""The development slice minus the deciding real-world slice, stated rather than left to be computed."""
|
|
||||||
from openultrasast.model.report import overfitting_gap, render
|
|
||||||
|
|
||||||
rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)]
|
|
||||||
gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js")
|
|
||||||
assert gap == pytest.approx(0.80 - 0.20)
|
|
||||||
assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower()
|
|
||||||
assert "+60" in render(rows, development="vibe-py", deciding="vfc-js")
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None:
|
|
||||||
"""A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one."""
|
|
||||||
from openultrasast.model.report import overfitting_gap
|
|
||||||
|
|
||||||
assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_every_finding_shows_its_rung() -> None:
|
|
||||||
from openultrasast.findings import StaticFinding
|
|
||||||
from openultrasast.model.ladder import Rung
|
|
||||||
from openultrasast.model.report import render_findings
|
|
||||||
|
|
||||||
finding = StaticFinding(
|
|
||||||
finding_id="r:app.py:1",
|
|
||||||
path="app.py",
|
|
||||||
title="t",
|
|
||||||
severity="high",
|
|
||||||
confidence="medium",
|
|
||||||
evidence_level="static_corroboration",
|
|
||||||
rationale="why",
|
|
||||||
line=1,
|
|
||||||
function_name="run",
|
|
||||||
reachability_status="unknown",
|
|
||||||
reachability_evidence=[],
|
|
||||||
reachability_conditions=[],
|
|
||||||
tags=[],
|
|
||||||
ranking_priority=1.0,
|
|
||||||
rung=Rung.ENTAILED,
|
|
||||||
)
|
|
||||||
text = render_findings([finding])
|
|
||||||
assert "model_entailed" in text and "app.py" in text
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_deciding_slice_is_named_in_the_report() -> None:
|
|
||||||
"""Req 9.3: the real-world slice decides, and the report says which one that is."""
|
|
||||||
from openultrasast.model.report import render
|
|
||||||
|
|
||||||
text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js")
|
|
||||||
assert "vfc-js" in text and "decid" in text.lower()
|
|
||||||
+43
-45
@@ -30,12 +30,12 @@ def _spec(): # type: ignore[no-untyped-def]
|
|||||||
|
|
||||||
def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
|
def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3}
|
{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="run")
|
answer = verdicts(_cpg(rows), _spec(), function="run")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
assert answer.family == "injection"
|
assert answer.family == "injection"
|
||||||
assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found"
|
assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found"
|
||||||
@@ -44,30 +44,30 @@ def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
|
|||||||
def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None:
|
def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None:
|
||||||
"""The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's."""
|
"""The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's."""
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}]
|
rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="run")
|
answer = verdicts(_cpg(rows), _spec(), function="run")[0]
|
||||||
assert answer is not None and answer.rung is Rung.CORROBORATED
|
assert answer is not None and answer.rung is Rung.CORROBORATED
|
||||||
|
|
||||||
|
|
||||||
def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None:
|
def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None:
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
assert verdict(_cpg([]), _spec(), function="run") is None
|
assert verdicts(_cpg([]), _spec(), function="run") == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None:
|
def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None:
|
||||||
"""`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3."""
|
"""`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3."""
|
||||||
from openultrasast.cpg.backend import CpgResult
|
from openultrasast.cpg.backend import CpgResult
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None)
|
broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None)
|
||||||
assert verdict(broken, _spec(), function="run") is None
|
assert verdicts(broken, _spec(), function="run") == []
|
||||||
|
|
||||||
|
|
||||||
def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
|
def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -79,20 +79,20 @@ def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
|
|||||||
"length": 2,
|
"length": 2,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
assert verdict(_cpg(rows), _spec(), function="run") is None, "a flow in another function does not judge this one"
|
assert verdicts(_cpg(rows), _spec(), function="run") == [], "a flow in another function does not judge this one"
|
||||||
|
|
||||||
|
|
||||||
def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None:
|
def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None:
|
||||||
"""Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average."""
|
"""Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average."""
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9},
|
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9},
|
||||||
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2},
|
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2},
|
||||||
]
|
]
|
||||||
first = verdict(_cpg(rows), _spec(), function="run")
|
first = verdicts(_cpg(rows), _spec(), function="run")[0]
|
||||||
second = verdict(_cpg(rows), _spec(), function="run")
|
second = verdicts(_cpg(rows), _spec(), function="run")[0]
|
||||||
assert first == second
|
assert first == second
|
||||||
assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness
|
assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@ def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic()
|
|||||||
def test_the_spec_drives_the_query_parameters() -> None:
|
def test_the_spec_drives_the_query_parameters() -> None:
|
||||||
"""The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam."""
|
"""The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam."""
|
||||||
from openultrasast.cpg.backend import CpgResult
|
from openultrasast.cpg.backend import CpgResult
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
seen: dict[str, object] = {}
|
seen: dict[str, object] = {}
|
||||||
|
|
||||||
@@ -109,7 +109,7 @@ def test_the_spec_drives_the_query_parameters() -> None:
|
|||||||
seen.update(params)
|
seen.update(params)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
verdict(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run")
|
verdicts(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run")
|
||||||
assert seen["query"] == "taint"
|
assert seen["query"] == "taint"
|
||||||
assert seen["sources"] == ("request.args",)
|
assert seen["sources"] == ("request.args",)
|
||||||
assert seen["sinks"] == ("os.system", "execute")
|
assert seen["sinks"] == ("os.system", "execute")
|
||||||
@@ -143,7 +143,7 @@ def _shape_spec(): # type: ignore[no-untyped-def]
|
|||||||
|
|
||||||
|
|
||||||
def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_reaches_it() -> None:
|
def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_reaches_it() -> None:
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -157,12 +157,12 @@ def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_r
|
|||||||
"sinkArg0Literal": True,
|
"sinkArg0Literal": True,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
assert verdict(_cpg(rows), _shape_spec(), function="run") is None, "a bound query is the fix, not the bug"
|
assert verdicts(_cpg(rows), _shape_spec(), function="run") == [], "a bound query is the fix, not the bug"
|
||||||
|
|
||||||
|
|
||||||
def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
|
def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -176,14 +176,14 @@ def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
|
|||||||
"sinkArg0Literal": False,
|
"sinkArg0Literal": False,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _shape_spec(), function="run")
|
answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
|
|
||||||
|
|
||||||
def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> None:
|
def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> None:
|
||||||
"""`os.system(x)` with one argument is not "safe because it has one argument" -- the rule is per sink."""
|
"""`os.system(x)` with one argument is not "safe because it has one argument" -- the rule is per sink."""
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -197,17 +197,17 @@ def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> N
|
|||||||
"sinkArg0Literal": False,
|
"sinkArg0Literal": False,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="run") # os.system is not in safe_shape_sinks
|
answer = verdicts(_cpg(rows), _spec(), function="run")[0] # os.system is not in safe_shape_sinks
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
|
|
||||||
|
|
||||||
def test_a_row_without_shape_fields_is_judged_on_the_flow_alone() -> None:
|
def test_a_row_without_shape_fields_is_judged_on_the_flow_alone() -> None:
|
||||||
"""Older rows, or an engine that could not report arity, must not be silently treated as safe."""
|
"""Older rows, or an engine that could not report arity, must not be silently treated as safe."""
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [{"sink": "db.execute(q)", "sinkLine": "5", "sinkMethod": "run", "source": "request.args['n']", "sanitized": False, "length": 3}]
|
rows = [{"sink": "db.execute(q)", "sinkLine": "5", "sinkMethod": "run", "source": "request.args['n']", "sanitized": False, "length": 3}]
|
||||||
answer = verdict(_cpg(rows), _shape_spec(), function="run")
|
answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
|
|
||||||
|
|
||||||
@@ -219,7 +219,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
|
|||||||
So the flag must exist and must default to off.
|
So the flag must exist and must default to off.
|
||||||
"""
|
"""
|
||||||
from openultrasast.cpg.backend import CpgResult
|
from openultrasast.cpg.backend import CpgResult
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
seen: dict[str, object] = {}
|
seen: dict[str, object] = {}
|
||||||
|
|
||||||
@@ -227,9 +227,9 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
|
|||||||
seen.update(params)
|
seen.update(params)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run")
|
verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run")
|
||||||
assert seen["parameterSources"] == "false", "treating every parameter as untrusted is not the default"
|
assert seen["parameterSources"] == "false", "treating every parameter as untrusted is not the default"
|
||||||
verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True)
|
verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True)
|
||||||
assert seen["parameterSources"] == "true"
|
assert seen["parameterSources"] == "true"
|
||||||
|
|
||||||
|
|
||||||
@@ -250,7 +250,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
|
|||||||
|
|
||||||
def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None:
|
def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None:
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -263,13 +263,13 @@ def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None
|
|||||||
"inLabeledScope": True,
|
"inLabeledScope": True,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="requestListener")
|
answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
assert "<lambda>0" not in answer.witness, "the witness names the flow, not the synthetic closure name"
|
assert "<lambda>0" not in answer.witness, "the witness names the flow, not the synthetic closure name"
|
||||||
|
|
||||||
|
|
||||||
def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
|
def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -282,17 +282,17 @@ def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
|
|||||||
"inLabeledScope": False,
|
"inLabeledScope": False,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
assert verdict(_cpg(rows), _spec(), function="requestListener") is None
|
assert verdicts(_cpg(rows), _spec(), function="requestListener") == []
|
||||||
|
|
||||||
|
|
||||||
def test_rows_without_the_scope_flag_fall_back_to_the_method_name() -> None:
|
def test_rows_without_the_scope_flag_fall_back_to_the_method_name() -> None:
|
||||||
"""A row from an engine that cannot report scope must not silently become in-scope."""
|
"""A row from an engine that cannot report scope must not silently become in-scope."""
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{"sink": "os.system(x)", "sinkLine": "9", "sinkMethod": "<lambda>0", "source": "request.args['x']", "sanitized": False, "length": 2}
|
{"sink": "os.system(x)", "sinkLine": "9", "sinkMethod": "<lambda>0", "source": "request.args['x']", "sanitized": False, "length": 2}
|
||||||
]
|
]
|
||||||
assert verdict(_cpg(rows), _spec(), function="run") is None
|
assert verdicts(_cpg(rows), _spec(), function="run") == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter() -> None:
|
def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter() -> None:
|
||||||
@@ -305,7 +305,7 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter(
|
|||||||
breaks ties, so the order stays total and the verdict deterministic.
|
breaks ties, so the order stays total and the verdict deterministic.
|
||||||
"""
|
"""
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -336,10 +336,10 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter(
|
|||||||
"inLabeledScope": True,
|
"inLabeledScope": True,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="requestListener")
|
answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
assert "request.args['f']" in answer.witness, f"witness named no real source: {answer.witness}"
|
assert "request.args['f']" in answer.witness, f"witness named no real source: {answer.witness}"
|
||||||
assert verdict(_cpg(rows), _spec(), function="requestListener") == answer # still deterministic
|
assert verdicts(_cpg(rows), _spec(), function="requestListener")[0] == answer # still deterministic
|
||||||
|
|
||||||
|
|
||||||
def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitized() -> None:
|
def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitized() -> None:
|
||||||
@@ -351,7 +351,7 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize
|
|||||||
modelled source reaches the sink, those flows are the evidence and the bare-parameter ones are ignored.
|
modelled source reaches the sink, those flows are the evidence and the bare-parameter ones are ignored.
|
||||||
"""
|
"""
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -373,14 +373,14 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize
|
|||||||
"inLabeledScope": True,
|
"inLabeledScope": True,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="requestListener")
|
answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
|
||||||
assert answer is not None and answer.rung is Rung.CORROBORATED, "a sanitized real flow is not an entailment"
|
assert answer is not None and answer.rung is Rung.CORROBORATED, "a sanitized real flow is not an entailment"
|
||||||
assert "request.args['f']" in answer.witness
|
assert "request.args['f']" in answer.witness
|
||||||
|
|
||||||
|
|
||||||
def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() -> None:
|
def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() -> None:
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
@@ -393,7 +393,7 @@ def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink()
|
|||||||
"inLabeledScope": True,
|
"inLabeledScope": True,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
answer = verdict(_cpg(rows), _spec(), function="run")
|
answer = verdicts(_cpg(rows), _spec(), function="run")[0]
|
||||||
assert answer is not None and answer.rung is Rung.ENTAILED
|
assert answer is not None and answer.rung is Rung.ENTAILED
|
||||||
|
|
||||||
|
|
||||||
@@ -443,13 +443,13 @@ def test_a_bound_that_governs_the_sink_lowers_the_rung() -> None:
|
|||||||
from openultrasast.cpg.backend import CpgResult
|
from openultrasast.cpg.backend import CpgResult
|
||||||
from openultrasast.model.ladder import Rung
|
from openultrasast.model.ladder import Rung
|
||||||
from openultrasast.model.specs import taint_specs
|
from openultrasast.model.specs import taint_specs
|
||||||
from openultrasast.model.taint import verdict
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
spec = taint_specs(language="c")["memory"]
|
spec = taint_specs(language="c")["memory"]
|
||||||
assert "strcpy" in spec.bounded_sinks, "a CWE-121 sink is dischargeable by a bound"
|
assert "strcpy" in spec.bounded_sinks, "a CWE-121 sink is dischargeable by a bound"
|
||||||
|
|
||||||
governed = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main")
|
governed = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main")[0]
|
||||||
ungoverned = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main")
|
ungoverned = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main")[0]
|
||||||
|
|
||||||
assert governed is not None and governed.rung is Rung.CORROBORATED
|
assert governed is not None and governed.rung is Rung.CORROBORATED
|
||||||
assert "governed by" in governed.witness, "the witness names the bound it found"
|
assert "governed by" in governed.witness, "the witness names the bound it found"
|
||||||
@@ -471,7 +471,7 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None:
|
|||||||
verdict, and `system` won it on flow length, so the SQL injection went unreported by construction."""
|
verdict, and `system` won it on flow length, so the SQL injection went unreported by construction."""
|
||||||
from openultrasast.cpg.backend import CpgResult
|
from openultrasast.cpg.backend import CpgResult
|
||||||
from openultrasast.model.specs import taint_specs
|
from openultrasast.model.specs import taint_specs
|
||||||
from openultrasast.model.taint import verdict, verdicts
|
from openultrasast.model.taint import verdicts
|
||||||
|
|
||||||
def row(sink, line, length): # type: ignore[no-untyped-def]
|
def row(sink, line, length): # type: ignore[no-untyped-def]
|
||||||
return {
|
return {
|
||||||
@@ -497,5 +497,3 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None:
|
|||||||
|
|
||||||
assert len(every) == 2, "two sink sites are two findings"
|
assert len(every) == 2, "two sink sites are two findings"
|
||||||
assert {v.location for v in every} == {"app.php:6:handler", "app.php:13:handler"}
|
assert {v.location for v in every} == {"app.php:6:handler", "app.php:13:handler"}
|
||||||
# The pair path is unchanged: the strongest is still what `verdict` returns.
|
|
||||||
assert every[0].location == verdict(cpg, spec, function="", file="app.php").location
|
|
||||||
|
|||||||
@@ -62,22 +62,6 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None
|
|||||||
assert taxonomy.family_of_cwe("CWE-99999") is None
|
assert taxonomy.family_of_cwe("CWE-99999") is None
|
||||||
|
|
||||||
|
|
||||||
def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None:
|
|
||||||
from openultrasast.model.taxonomy import load_families
|
|
||||||
|
|
||||||
taxonomy = load_families()
|
|
||||||
assert taxonomy.related("injection", "injection") == "same"
|
|
||||||
assert taxonomy.related("injection", "access_control") == "lateral"
|
|
||||||
assert taxonomy.related("injection", "not_a_family") == "fabricated"
|
|
||||||
assert taxonomy.related("unknown", "injection") == "lateral" # abstention earns no partial credit
|
|
||||||
assert all(family.parent is None for family in taxonomy.families) # the shipped taxonomy is flat
|
|
||||||
(tmp_path / "families.toml").write_text(
|
|
||||||
GOOD + '\n[[family]]\nid = "path"\nparent = "injection"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "canary"\n'
|
|
||||||
)
|
|
||||||
scoped = load_families(tmp_path / "families.toml", require_all=False)
|
|
||||||
assert scoped.related("path", "injection") == "parent_child" and scoped.related("injection", "path") == "parent_child"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("text", "needle"),
|
("text", "needle"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -129,16 +129,6 @@ def test_state_is_mirrored_and_seeds_a_rerun_elsewhere(store: HttpObjectStore, t
|
|||||||
assert second.load_state() is None
|
assert second.load_state() is None
|
||||||
|
|
||||||
|
|
||||||
def test_put_input_stores_a_file_no_task_produces(store: HttpObjectStore, tmp_path: Path) -> None:
|
|
||||||
delivery = Delivery(store, "r", tmp_path / "r")
|
|
||||||
source = tmp_path / "candidates.json"
|
|
||||||
source.write_bytes(b"[1, 2, 3]")
|
|
||||||
url = delivery.put_input("candidates.json", source)
|
|
||||||
assert store.objects["runs/r/inputs/candidates.json"] == b"[1, 2, 3]"
|
|
||||||
with urllib.request.urlopen(url, timeout=10) as response: # noqa: S310
|
|
||||||
assert response.read() == b"[1, 2, 3]"
|
|
||||||
|
|
||||||
|
|
||||||
def test_declared_artifacts_are_outputs_plus_what_consumers_name(tmp_path: Path) -> None:
|
def test_declared_artifacts_are_outputs_plus_what_consumers_name(tmp_path: Path) -> None:
|
||||||
"""A consumer may declare a producer's summary.json (every task writes one) without the producer listing it."""
|
"""A consumer may declare a producer's summary.json (every task writes one) without the producer listing it."""
|
||||||
from openultrasast.plane.manifests import load_manifests
|
from openultrasast.plane.manifests import load_manifests
|
||||||
|
|||||||
@@ -3,8 +3,7 @@
|
|||||||
Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the memory store's
|
Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the memory store's
|
||||||
host (where the runner PUTs its tar and GETs its inputs), the bound Workspaces' Git hosts and the bound Model's API
|
host (where the runner PUTs its tar and GETs its inputs), the bound Workspaces' Git hosts and the bound Model's API
|
||||||
host, all as one TLS passthrough rule on 443 (plane-on-kubernetes 2.3), and nothing a gateway would reject or that
|
host, all as one TLS passthrough rule on 443 (plane-on-kubernetes 2.3), and nothing a gateway would reject or that
|
||||||
would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule (the receiver path, kept for
|
would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule.
|
||||||
group 2 only, is the one exception).
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -12,12 +11,11 @@ from __future__ import annotations
|
|||||||
import ipaddress
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
import threading
|
import threading
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from openultrasast.plane import egress, runner
|
from openultrasast.plane import egress
|
||||||
from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest
|
from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest
|
||||||
|
|
||||||
MANIFESTS = """
|
MANIFESTS = """
|
||||||
@@ -63,7 +61,7 @@ def hostnames(policy: dict) -> list[str]:
|
|||||||
|
|
||||||
def assert_tight(policy: dict) -> None:
|
def assert_tight(policy: dict) -> None:
|
||||||
for rule in policy["rules"]:
|
for rule in policy["rules"]:
|
||||||
assert set(rule) <= {"http", "tlsPassthrough"}, f"only http and passthrough rules, got {rule}"
|
assert set(rule) == {"tlsPassthrough"}, f"only TLS passthrough rules, got {rule}"
|
||||||
body = next(iter(rule.values()))
|
body = next(iter(rule.values()))
|
||||||
assert "all" not in body["ports"], "never every port"
|
assert "all" not in body["ports"], "never every port"
|
||||||
for host in hostnames(policy):
|
for host in hostnames(policy):
|
||||||
@@ -82,12 +80,6 @@ def test_model_free_task_gets_only_the_store(manifests) -> None:
|
|||||||
assert egress.policy_for(manifests.tasks["facts"], [], None, None) == {"rules": []}, "no store, no Git, no Model: nothing"
|
assert egress.policy_for(manifests.tasks["facts"], [], None, None) == {"rules": []}, "no store, no Git, no Model: nothing"
|
||||||
|
|
||||||
|
|
||||||
def test_the_receiver_path_adds_the_one_http_rule_until_it_is_removed(manifests) -> None:
|
|
||||||
policy = egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host=egress.RECEIVER_HOST)
|
|
||||||
assert policy == {"rules": [{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}}]}
|
|
||||||
assert egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host="127.0.0.1") == {"rules": []}
|
|
||||||
|
|
||||||
|
|
||||||
def test_model_bound_task_with_git_workspaces(manifests) -> None:
|
def test_model_bound_task_with_git_workspaces(manifests) -> None:
|
||||||
task = manifests.tasks["verify"]
|
task = manifests.tasks["verify"]
|
||||||
policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], STORE)
|
policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], STORE)
|
||||||
@@ -191,50 +183,3 @@ def test_apply_waits_for_the_actor_then_gives_up(tmp_path: Path, kubectl_ate: Pa
|
|||||||
timer.start()
|
timer.start()
|
||||||
cli.apply("late", "default", POLICY, timeout=10)
|
cli.apply("late", "default", POLICY, timeout=10)
|
||||||
timer.join()
|
timer.join()
|
||||||
|
|
||||||
|
|
||||||
def test_receiver_address_modes(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
|
|
||||||
assert egress.receiver_address(4242, None) == ("http://127.0.0.1:4242/", None)
|
|
||||||
monkeypatch.delenv("OUSAST_ARTIFACT_HOST")
|
|
||||||
monkeypatch.setenv("OUSAST_ARTIFACT_DIAL", "172.19.0.1:80")
|
|
||||||
assert egress.receiver_address(18090, "kind-test") == (f"http://{egress.RECEIVER_HOST}/", "172.19.0.1:80")
|
|
||||||
monkeypatch.delenv("OUSAST_ARTIFACT_DIAL")
|
|
||||||
with pytest.raises(RuntimeError, match="kube context"):
|
|
||||||
egress.receiver_address(18090, None) # no context to look the Service up in: never a built one
|
|
||||||
|
|
||||||
|
|
||||||
# --- the runner dials the gateway address with the receiver's name as Host ----------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
class _Seen(ThreadingHTTPServer):
|
|
||||||
daemon_threads = True
|
|
||||||
seen: list[tuple[str, str | None, bytes]]
|
|
||||||
|
|
||||||
|
|
||||||
class _Handler(BaseHTTPRequestHandler):
|
|
||||||
server: _Seen
|
|
||||||
|
|
||||||
def do_POST(self) -> None: # noqa: N802
|
|
||||||
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
|
|
||||||
self.server.seen.append((self.path, self.headers.get("Host"), body))
|
|
||||||
self.send_response(200)
|
|
||||||
self.send_header("Content-Length", "0")
|
|
||||||
self.end_headers()
|
|
||||||
|
|
||||||
def log_message(self, format: str, *args: object) -> None: # noqa: A002
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def test_delivery_dials_the_override_with_the_url_host() -> None:
|
|
||||||
server = _Seen(("127.0.0.1", 0), _Handler)
|
|
||||||
server.seen = []
|
|
||||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
||||||
try:
|
|
||||||
dial = f"127.0.0.1:{server.server_address[1]}"
|
|
||||||
url = f"http://{egress.RECEIVER_HOST}/in"
|
|
||||||
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, dial=dial) == 200
|
|
||||||
assert server.seen == [("/in", egress.RECEIVER_HOST, b"tar")]
|
|
||||||
finally:
|
|
||||||
server.shutdown()
|
|
||||||
server.server_close()
|
|
||||||
|
|||||||
@@ -9,10 +9,9 @@ from pathlib import Path
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from openultrasast.learn.schema import FeatureRecordError, feature_set_digest
|
|
||||||
from openultrasast.plane.memory import FileStore
|
from openultrasast.plane.memory import FileStore
|
||||||
from openultrasast.plane.tasks import features
|
from openultrasast.plane.tasks import features
|
||||||
from openultrasast.plane.tasks.remember import Context, remember_run, rows_for
|
from openultrasast.plane.tasks.remember import remember_run
|
||||||
|
|
||||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
||||||
CASE = "lmdeploy-media-url-ssrf"
|
CASE = "lmdeploy-media-url-ssrf"
|
||||||
@@ -25,30 +24,6 @@ def run_dir(tmp_path: Path) -> Path:
|
|||||||
return target
|
return target
|
||||||
|
|
||||||
|
|
||||||
def _checkout(root: Path) -> Path:
|
|
||||||
media = root / "lmdeploy" / "vl" / "media"
|
|
||||||
media.mkdir(parents=True)
|
|
||||||
(media / "connection.py").write_text("def _is_safe_url(u):\n return True\n\n\ndef _load_http_url(u):\n x = 1\n return x\n")
|
|
||||||
return root
|
|
||||||
|
|
||||||
|
|
||||||
def test_features_of_the_recorded_case(run_dir: Path, tmp_path: Path) -> None:
|
|
||||||
[rec] = features.recorded_case(run_dir, CASE, workspace=_checkout(tmp_path / "checkout"))
|
|
||||||
assert (rec["candidate"], rec["family"], rec["profile"], rec["unit"]) == (
|
|
||||||
"lmdeploy/vl/media/connection.py::_load_http_url", "untrusted_destination", "plane", "pin",
|
|
||||||
) # fmt: skip
|
|
||||||
x, states = rec["x"], {k: v["state"] for k, v in rec["instruments"].items()}
|
|
||||||
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, True, None, 2)
|
|
||||||
assert x["agree.final"] == "agreed" and x["facts.callers"] == 0 and x["facts.is_global"] is False
|
|
||||||
assert x["facts.function_lines"] == 3 and x["lang.python"] is True and x["verify.site_offset"] == 0
|
|
||||||
assert states["verify"] == states["agree"] == states["facts"] == states["source"] == "ran"
|
|
||||||
assert states["quick"] == states["engine"] == "none", "no alerts task in this Run: no coverage, not zero hits"
|
|
||||||
assert rec["instruments"]["verify"]["version"] == "deepseek-flash" and rec["feature_set_digest"] == feature_set_digest()
|
|
||||||
assert "site_match" not in json.dumps(rec), "the fixture's site_match: true was dropped on read"
|
|
||||||
[without] = features.recorded_case(run_dir, CASE)
|
|
||||||
assert without["instruments"]["source"]["state"] == "none" and without["x"]["facts.function_lines"] is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None:
|
def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None:
|
||||||
out = tmp_path / "out"
|
out = tmp_path / "out"
|
||||||
env = {
|
env = {
|
||||||
@@ -71,13 +46,5 @@ def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_pat
|
|||||||
assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed"
|
assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed"
|
||||||
|
|
||||||
|
|
||||||
def test_remember_refuses_a_record_outside_the_allow_list(run_dir: Path) -> None:
|
|
||||||
[rec] = features.recorded_case(run_dir, CASE)
|
|
||||||
rec["x"]["site_match"] = True
|
|
||||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
|
||||||
with pytest.raises(FeatureRecordError, match="site_match"):
|
|
||||||
rows_for(ctx, facts=None, passes={}, models={}, agreed=None, features=[rec])
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_task_fails_loudly_without_an_output_dir() -> None:
|
def test_the_task_fails_loudly_without_an_output_dir() -> None:
|
||||||
assert features.main({}) == 2
|
assert features.main({}) == 2
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
|
|||||||
assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls
|
assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls
|
||||||
env = {e.name: e.value for e in task.env}
|
env = {e.name: e.value for e in task.env}
|
||||||
assert env["OUSAST_PASS"] == entry.name[-1]
|
assert env["OUSAST_PASS"] == entry.name[-1]
|
||||||
docs = reconciler.render_task(run, entry, manifests, "http://127.0.0.1:1/")
|
docs = reconciler.render_task(run, entry, manifests)
|
||||||
for doc in docs:
|
for doc in docs:
|
||||||
assert not non_ax_fields(doc["spec"], AX_FIELDS[doc["kind"]], doc["kind"])
|
assert not non_ax_fields(doc["spec"], AX_FIELDS[doc["kind"]], doc["kind"])
|
||||||
assert set(doc["metadata"]) <= {"name", "atespace"}
|
assert set(doc["metadata"]) <= {"name", "atespace"}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates
|
|||||||
fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
|
fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
|
||||||
image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
|
image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
|
||||||
-- and on the scripted poll it PUTs a scripted tar to the presigned ``delivery.put`` URL the start request carried
|
-- and on the scripted poll it PUTs a scripted tar to the presigned ``delivery.put`` URL the start request carried
|
||||||
(plane-on-kubernetes 2.4; or POSTs it to ``OUSAST_ARTIFACT_URL`` in receiver mode), or turns ``Failed`` with a
|
(plane-on-kubernetes 2.4), or turns ``Failed`` with a
|
||||||
condition message, or vanishes. Every outcome comes from a scripted delivery, and a delivery happens only after
|
condition message, or vanishes. Every outcome comes from a scripted delivery, and a delivery happens only after
|
||||||
the reconciler's start request reached the fake router (Req 4.5). The store is ``plane_fake_store.HttpObjectStore``.
|
the reconciler's start request reached the fake router (Req 4.5). The store is ``plane_fake_store.HttpObjectStore``.
|
||||||
"""
|
"""
|
||||||
@@ -21,8 +21,6 @@ import sys
|
|||||||
import tarfile
|
import tarfile
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -152,11 +150,9 @@ elif argv[:2] == ["get", "task"]:
|
|||||||
delivery = body.get("delivery") or {{}}
|
delivery = body.get("delivery") or {{}}
|
||||||
if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
|
if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
|
||||||
files = script.get("files", {{}})
|
files = script.get("files", {{}})
|
||||||
base = env.get("OUSAST_ARTIFACT_URL", "").rstrip("/") + "/inputs/"
|
|
||||||
for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items():
|
for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items():
|
||||||
presigned = (delivery.get("inputs") or {{}}).get(name)
|
presigned = (delivery.get("inputs") or {{}}).get(name)
|
||||||
fetch = urllib.request.Request(presigned) if presigned else urllib.request.Request(
|
fetch = urllib.request.Request(presigned)
|
||||||
base + ref, headers={{"X-Ousast-Run": env["OUSAST_RUN"], "X-Ousast-Task": task}})
|
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(fetch) as resp:
|
with urllib.request.urlopen(fetch) as resp:
|
||||||
dest = HERE / "fetched" / task / ref
|
dest = HERE / "fetched" / task / ref
|
||||||
@@ -175,13 +171,8 @@ elif argv[:2] == ["get", "task"]:
|
|||||||
info = tarfile.TarInfo(path)
|
info = tarfile.TarInfo(path)
|
||||||
info.size = len(data)
|
info.size = len(data)
|
||||||
tar.addfile(info, io.BytesIO(data))
|
tar.addfile(info, io.BytesIO(data))
|
||||||
headers = {{"Content-Type": "application/x-tar", "X-Ousast-Run": env["OUSAST_RUN"]}}
|
|
||||||
headers["X-Ousast-Task"] = script.get("as_task", task)
|
|
||||||
if delivery.get("put"): # the store path: PUT to the presigned URL, no receiver headers
|
|
||||||
tar_headers = {{"Content-Type": "application/x-tar"}}
|
tar_headers = {{"Content-Type": "application/x-tar"}}
|
||||||
req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers)
|
req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers)
|
||||||
else:
|
|
||||||
req = urllib.request.Request(env["OUSAST_ARTIFACT_URL"], data=buf.getvalue(), method="POST", headers=headers)
|
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req) as resp:
|
with urllib.request.urlopen(req) as resp:
|
||||||
(HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n")
|
(HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n")
|
||||||
@@ -420,7 +411,6 @@ def test_chain_runs_in_order_with_inputs_env_and_model(fake: Fake, tmp_path: Pat
|
|||||||
assert env["OUSAST_BUDGET_USD"] == "2.5" and env["OUSAST_BUDGET_CALLS"] == "40"
|
assert env["OUSAST_BUDGET_USD"] == "2.5" and env["OUSAST_BUDGET_CALLS"] == "40"
|
||||||
assert env["OUSAST_INPUT_FACTS"] == "/workspace/.ousast-in/verify/repo-facts/facts.json"
|
assert env["OUSAST_INPUT_FACTS"] == "/workspace/.ousast-in/verify/repo-facts/facts.json"
|
||||||
assert json.loads(env["OUSAST_INPUTS"]) == {"FACTS": "repo-facts/facts.json"}
|
assert json.loads(env["OUSAST_INPUTS"]) == {"FACTS": "repo-facts/facts.json"}
|
||||||
assert "OUSAST_ARTIFACT_URL" not in env, "the store path renders no receiver address into the Task"
|
|
||||||
assert env["OUSAST_MODEL"] == "deepseek-flash"
|
assert env["OUSAST_MODEL"] == "deepseek-flash"
|
||||||
assert json.loads(env["OUSAST_MODEL_PARAMS"]) == {"cache_hit_per_m": 0.07, "input_per_m": 0.27, "output_per_m": 1.1}
|
assert json.loads(env["OUSAST_MODEL_PARAMS"]) == {"cache_hit_per_m": 0.07, "input_per_m": 0.27, "output_per_m": 1.1}
|
||||||
kinds = [(d["kind"], d["metadata"]["name"]) for d in verify["docs"]]
|
kinds = [(d["kind"], d["metadata"]["name"]) for d in verify["docs"]]
|
||||||
@@ -661,33 +651,6 @@ def tar_of(files: dict[str, bytes]) -> bytes:
|
|||||||
return buf.getvalue()
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
def test_receiver_rejects_traversal_unknown_tasks_and_wrong_types(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") # the direct test address: no Service lookup, no context needed
|
|
||||||
receiver = reconciler.Receiver("r", tmp_path / "r", 0)
|
|
||||||
receiver.running.add("t")
|
|
||||||
headers = {"X-Ousast-Run": "r", "X-Ousast-Task": "t", "Content-Type": "application/x-tar"}
|
|
||||||
code, text = receiver.accept(headers, tar_of({"../escape.json": b"{}"}))
|
|
||||||
assert code == 400 and "escape" in text and not (tmp_path / "escape.json").exists()
|
|
||||||
assert receiver.accept({**headers, "X-Ousast-Task": "other"}, tar_of({"summary.json": b"{}"}))[0] == 403
|
|
||||||
assert receiver.accept({**headers, "X-Ousast-Run": "someone"}, tar_of({"summary.json": b"{}"}))[0] == 403
|
|
||||||
assert receiver.accept({**headers, "Content-Type": "application/json"}, b"{}")[0] == 415
|
|
||||||
assert receiver.accept(headers, b"not a tar")[0] == 400
|
|
||||||
assert receiver.accept(headers, tar_of({"summary.json": b"{}", "sub/x.json": b"1"}))[0] == 200
|
|
||||||
assert (tmp_path / "r" / "t" / "sub" / "x.json").read_text() == "1" and "t" in receiver.delivered
|
|
||||||
receiver.server_close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_delivery_for_a_task_that_is_not_running_is_rejected(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
monkeypatch.setenv("OUSAST_DELIVERY", "receiver") # the receiver path, kept for group 2 (task 2.6 removes this test)
|
|
||||||
monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "1") # the rejected delivery is no delivery: the task times out
|
|
||||||
script = json.loads(json.dumps(SCRIPT))
|
|
||||||
script["repo-facts"]["as_task"] = "agree" # the runner claims to be a task that has not started
|
|
||||||
fake.script(script)
|
|
||||||
assert reconciler.run(write_run(tmp_path, "spoof"), ax=str(fake.ax)) == "failed"
|
|
||||||
assert [n for _, n in fake.events("rejected")] == ["repo-facts"]
|
|
||||||
assert not (reconciler.run_dir("spoof") / "agree" / "summary.json").exists()
|
|
||||||
|
|
||||||
|
|
||||||
def test_lock_refuses_a_second_runner_and_ignores_a_dead_one(fake: Fake, tmp_path: Path) -> None:
|
def test_lock_refuses_a_second_runner_and_ignores_a_dead_one(fake: Fake, tmp_path: Path) -> None:
|
||||||
manifest = write_run(tmp_path, "locked")
|
manifest = write_run(tmp_path, "locked")
|
||||||
base = reconciler.run_dir("locked")
|
base = reconciler.run_dir("locked")
|
||||||
@@ -824,7 +787,7 @@ def test_rendered_documents_carry_only_ax_fields(tmp_path: Path) -> None:
|
|||||||
(tmp_path / "repo-facts").mkdir()
|
(tmp_path / "repo-facts").mkdir()
|
||||||
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
|
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
|
||||||
run, manifests = reconciler.load_run(write_run(tmp_path, "fields"))
|
run, manifests = reconciler.load_run(write_run(tmp_path, "fields"))
|
||||||
docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests, "http://h:1/")]
|
docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests)]
|
||||||
assert {d["kind"] for d in docs} == {"Task", "Workspace", "Model"}
|
assert {d["kind"] for d in docs} == {"Task", "Workspace", "Model"}
|
||||||
for rendered in docs:
|
for rendered in docs:
|
||||||
where = f"{rendered['kind']}/{rendered['metadata']['name']}"
|
where = f"{rendered['kind']}/{rendered['metadata']['name']}"
|
||||||
@@ -841,7 +804,7 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None:
|
|||||||
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
|
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
|
||||||
run, manifests = reconciler.load_run(write_run(tmp_path, "order"))
|
run, manifests = reconciler.load_run(write_run(tmp_path, "order"))
|
||||||
for entry in run.tasks:
|
for entry in run.tasks:
|
||||||
docs = reconciler.render_task(run, entry, manifests, "http://h:1/")
|
docs = reconciler.render_task(run, entry, manifests)
|
||||||
assert [d["kind"] for d in docs].count("Task") == 1
|
assert [d["kind"] for d in docs].count("Task") == 1
|
||||||
assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs]
|
assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs]
|
||||||
|
|
||||||
@@ -943,44 +906,6 @@ def test_a_refused_egress_policy_fails_the_task_before_resume(fake: Fake, tmp_pa
|
|||||||
SUBSTRATE_ENV_LIMIT = 32768 # "actor_template.containers[0].env[11].value: Too long" on the live cluster, 2026-09-29
|
SUBSTRATE_ENV_LIMIT = 32768 # "actor_template.containers[0].env[11].value: Too long" on the live cluster, 2026-09-29
|
||||||
|
|
||||||
|
|
||||||
def get_input(receiver: reconciler.Receiver, path: str, run: str = "r", task: str = "t") -> tuple[int, bytes]:
|
|
||||||
url = f"http://127.0.0.1:{receiver.server_address[1]}{path}"
|
|
||||||
request = urllib.request.Request(url, headers={"X-Ousast-Run": run, "X-Ousast-Task": task})
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(request, timeout=10) as response:
|
|
||||||
return response.status, response.read()
|
|
||||||
except urllib.error.HTTPError as exc:
|
|
||||||
return exc.code, exc.read()
|
|
||||||
|
|
||||||
|
|
||||||
def test_receiver_serves_a_running_task_its_declared_inputs_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
|
|
||||||
base = tmp_path / "r"
|
|
||||||
(base / "p").mkdir(parents=True)
|
|
||||||
big = b"x" * (3 << 20)
|
|
||||||
(base / "p" / "a.json").write_bytes(big)
|
|
||||||
(base / "p" / "b.json").write_bytes(b"not declared")
|
|
||||||
(tmp_path / "secret.txt").write_text("outside the run")
|
|
||||||
receiver = reconciler.Receiver("r", base, 0)
|
|
||||||
threading.Thread(target=receiver.serve_forever, daemon=True).start()
|
|
||||||
try:
|
|
||||||
receiver.running.update({"t", "idle"})
|
|
||||||
receiver.inputs.update({"t": frozenset({"p/a.json", "p/missing.json", "../secret.txt"}), "done": frozenset({"p/a.json"})})
|
|
||||||
assert get_input(receiver, "/inputs/p/a.json") == (200, big), "streamed whole"
|
|
||||||
assert get_input(receiver, "/inputs/p/a.json", run="other")[0] == 403, "wrong run"
|
|
||||||
assert get_input(receiver, "/inputs/p/a.json", task="done")[0] == 403, "a task that is not running"
|
|
||||||
assert get_input(receiver, "/inputs/p/a.json", task="idle")[0] == 403, "a running task without that input"
|
|
||||||
assert get_input(receiver, "/inputs/p/b.json")[0] == 403, "not among the task's inputs"
|
|
||||||
assert get_input(receiver, "/inputs/p/missing.json")[0] == 404, "declared but not delivered"
|
|
||||||
for escape in ("/inputs/../secret.txt", "/inputs/%2e%2e/secret.txt", "/inputs/p/../../secret.txt", "/inputs//etc/passwd"):
|
|
||||||
code, body = get_input(receiver, escape)
|
|
||||||
assert code in (403, 404) and b"outside" not in body, escape
|
|
||||||
assert get_input(receiver, "/p/a.json")[0] == 404, "only /inputs/ is served"
|
|
||||||
finally:
|
|
||||||
receiver.shutdown()
|
|
||||||
receiver.server_close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_missing_producer_artifact_fails_the_consumer_instead_of_running_it(fake: Fake, tmp_path: Path) -> None:
|
def test_a_missing_producer_artifact_fails_the_consumer_instead_of_running_it(fake: Fake, tmp_path: Path) -> None:
|
||||||
script = json.loads(json.dumps(SCRIPT))
|
script = json.loads(json.dumps(SCRIPT))
|
||||||
del script["repo-facts"]["files"]["facts.json"] # repo-facts reports done but never delivered facts.json
|
del script["repo-facts"]["files"]["facts.json"] # repo-facts reports done but never delivered facts.json
|
||||||
@@ -996,7 +921,7 @@ def test_a_megabyte_producer_artifact_keeps_every_rendered_value_under_substrate
|
|||||||
(tmp_path / "repo-facts" / "facts.json").write_text(json.dumps({"blob": "f" * (1 << 20)}), encoding="utf-8")
|
(tmp_path / "repo-facts" / "facts.json").write_text(json.dumps({"blob": "f" * (1 << 20)}), encoding="utf-8")
|
||||||
run, manifests = reconciler.load_run(write_run(tmp_path, "big"))
|
run, manifests = reconciler.load_run(write_run(tmp_path, "big"))
|
||||||
for entry in run.tasks:
|
for entry in run.tasks:
|
||||||
docs = reconciler.render_task(run, entry, manifests, "http://h:1/", "10.0.0.1:80")
|
docs = reconciler.render_task(run, entry, manifests)
|
||||||
task, workspaces = docs[-1], [d for d in docs if d["kind"] == "Workspace"]
|
task, workspaces = docs[-1], [d for d in docs if d["kind"] == "Workspace"]
|
||||||
assert all(len(e["value"]) < SUBSTRATE_ENV_LIMIT for e in task["spec"]["env"]), entry.name
|
assert all(len(e["value"]) < SUBSTRATE_ENV_LIMIT for e in task["spec"]["env"]), entry.name
|
||||||
assert len(yaml.safe_dump(task)) < SUBSTRATE_ENV_LIMIT, f"{entry.name}: AX_TASK_YAML"
|
assert len(yaml.safe_dump(task)) < SUBSTRATE_ENV_LIMIT, f"{entry.name}: AX_TASK_YAML"
|
||||||
@@ -1080,15 +1005,3 @@ def test_a_tar_the_store_path_cannot_accept_fails_the_task(fake: Fake, tmp_path:
|
|||||||
assert not (reconciler.run_dir("esc").parent / "escape.json").exists()
|
assert not (reconciler.run_dir("esc").parent / "escape.json").exists()
|
||||||
|
|
||||||
|
|
||||||
def test_receiver_mode_still_runs_through_the_host_receiver(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
"""OUSAST_DELIVERY=receiver (group 2 only): the old path, with its http rule and OUSAST_ARTIFACT_URL, until 2.6."""
|
|
||||||
monkeypatch.setenv("OUSAST_DELIVERY", "receiver")
|
|
||||||
assert reconciler.run(write_run(tmp_path, "rcv"), ax=str(fake.ax)) == "done"
|
|
||||||
assert fake.applied("rcv", "verify")["env"]["OUSAST_ARTIFACT_URL"].startswith("http://127.0.0.1:")
|
|
||||||
assert [s["body"].get("delivery") for s in fake.router.starts] == [None, None, None], "no presigned URLs in receiver mode"
|
|
||||||
assert not any(k.startswith("runs/rcv/") and k.endswith("output.tar") for k in fake.store.objects), "nothing PUT to the store"
|
|
||||||
assert "runs/rcv/state.json" in fake.store.objects, "the state is still mirrored"
|
|
||||||
written = json.loads((tmp_path / "policies" / "default_rcv-repo-facts.written.json").read_text())
|
|
||||||
assert written["rules"] == [{"tlsPassthrough": {"hostnames": ["example.invalid"], "ports": {"numbers": [443]}}}], (
|
|
||||||
"an IP receiver: no http rule"
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]:
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
|
def test_remember_turns_the_saved_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
|
||||||
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||||
assert len(facts) > 100, "the fixture was read"
|
assert len(facts) > 100, "the fixture was read"
|
||||||
store = FileStore(tmp_path / "memory")
|
store = FileStore(tmp_path / "memory")
|
||||||
|
|||||||
+29
-217
@@ -19,13 +19,12 @@ import time
|
|||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from plane_fake_store import HttpObjectStore
|
from plane_fake_store import HttpObjectStore
|
||||||
|
|
||||||
from openultrasast.plane import egress, runner
|
from openultrasast.plane import runner
|
||||||
from openultrasast.plane.manifests import GitSource
|
from openultrasast.plane.manifests import GitSource
|
||||||
from openultrasast.plane.router import Router, StartError
|
from openultrasast.plane.router import Router, StartError
|
||||||
from openultrasast.plane.runner import (
|
from openultrasast.plane.runner import (
|
||||||
@@ -194,96 +193,10 @@ def test_main_serves_readiness_on_a_free_port_and_the_task_sees_200(
|
|||||||
# --- artifact delivery (Requirement 4.4) --------------------------------------------------------------------------------
|
# --- artifact delivery (Requirement 4.4) --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class Receiver(ThreadingHTTPServer):
|
|
||||||
"""Records every POST (headers and body) and answers with ``status``."""
|
|
||||||
|
|
||||||
def __init__(self, status: int) -> None:
|
|
||||||
super().__init__(("127.0.0.1", 0), _ReceiverHandler)
|
|
||||||
self.status = status
|
|
||||||
self.posts: list[tuple[dict[str, str], bytes]] = []
|
|
||||||
self.thread = threading.Thread(target=self.serve_forever, daemon=True)
|
|
||||||
self.thread.start()
|
|
||||||
|
|
||||||
@property
|
|
||||||
def url(self) -> str:
|
|
||||||
return f"http://127.0.0.1:{self.server_address[1]}/artifacts"
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
|
||||||
self.shutdown()
|
|
||||||
self.server_close()
|
|
||||||
|
|
||||||
|
|
||||||
class _ReceiverHandler(BaseHTTPRequestHandler):
|
|
||||||
server: Receiver
|
|
||||||
|
|
||||||
def do_POST(self) -> None: # noqa: N802
|
|
||||||
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
||||||
self.server.posts.append(({k: v for k, v in self.headers.items()}, body))
|
|
||||||
self.send_response(self.server.status)
|
|
||||||
self.send_header("Content-Length", "0")
|
|
||||||
self.end_headers()
|
|
||||||
|
|
||||||
def log_message(self, format: str, *args: object) -> None: # noqa: A002
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def receiver() -> Iterator[Receiver]:
|
|
||||||
server = Receiver(200)
|
|
||||||
try:
|
|
||||||
yield server
|
|
||||||
finally:
|
|
||||||
server.stop()
|
|
||||||
|
|
||||||
|
|
||||||
def test_output_dir_is_delivered_as_a_tar_with_the_headers(
|
|
||||||
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
||||||
) -> None:
|
|
||||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, OUSAST_RUN="validation-46")
|
|
||||||
assert code == 0
|
|
||||||
assert len(receiver.posts) == 1
|
|
||||||
headers, body = receiver.posts[0]
|
|
||||||
assert headers["Content-Type"] == "application/x-tar"
|
|
||||||
assert headers["X-Ousast-Task"] == "stub"
|
|
||||||
assert headers["X-Ousast-Run"] == "validation-46"
|
|
||||||
receiver.posts.clear()
|
|
||||||
again = {"OUSAST_ARTIFACT_URL": receiver.url, "OUSAST_TASK": "facts", "OUSAST_STATE_DIR": str(tmp_path / "again-state")}
|
|
||||||
code, _ = run_main(monkeypatch, tmp_path / "again", files_workspace_yaml(), **again)
|
|
||||||
assert code == 0 and receiver.posts[0][0]["X-Ousast-Task"] == "facts", "the Run's task name, not the ax Task's"
|
|
||||||
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as archive:
|
|
||||||
names = sorted(archive.getnames())
|
|
||||||
summary = archive.extractfile("summary.json")
|
|
||||||
assert summary is not None
|
|
||||||
assert json.loads(summary.read())["status"] == "done"
|
|
||||||
assert names == ["facts.json", "summary.json"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_run_header_is_absent_without_ousast_run(
|
|
||||||
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
||||||
) -> None:
|
|
||||||
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url)
|
|
||||||
assert code == 0
|
|
||||||
assert "X-Ousast-Run" not in receiver.posts[0][0]
|
|
||||||
|
|
||||||
|
|
||||||
def test_delivery_failure_is_reported_as_failed_with_exit_2(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
||||||
broken = Receiver(500)
|
|
||||||
try:
|
|
||||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url)
|
|
||||||
finally:
|
|
||||||
broken.stop()
|
|
||||||
assert code == EXIT_BY_STATUS["failed"] == 2
|
|
||||||
assert len(broken.posts) == 3, "three attempts"
|
|
||||||
summary = json.loads((output / "summary.json").read_text())
|
|
||||||
assert summary["status"] == "failed"
|
|
||||||
assert summary["reason"].startswith("artifact delivery failed: HTTP 500")
|
|
||||||
assert summary["units_done"] == 1 and summary["units_total"] == 1, "the task's other fields are kept"
|
|
||||||
|
|
||||||
|
|
||||||
def test_deliver_retries_on_connection_errors_then_raises() -> None:
|
def test_deliver_retries_on_connection_errors_then_raises() -> None:
|
||||||
closed = Receiver(200)
|
closed = HttpObjectStore()
|
||||||
url = closed.url
|
url = closed.presign_put("output.tar")
|
||||||
closed.stop()
|
closed.close()
|
||||||
with pytest.raises(DeliveryError, match="3 attempts"):
|
with pytest.raises(DeliveryError, match="3 attempts"):
|
||||||
deliver(url, b"x", {"Content-Type": "application/x-tar"}, backoff=0)
|
deliver(url, b"x", {"Content-Type": "application/x-tar"}, backoff=0)
|
||||||
|
|
||||||
@@ -366,9 +279,9 @@ def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name:
|
|||||||
return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
|
return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
|
||||||
|
|
||||||
|
|
||||||
def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
|
def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], tmp_path: Path) -> None:
|
||||||
port = free_port()
|
port = free_port()
|
||||||
proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url)
|
proc = spawn_runner(tmp_path, port)
|
||||||
marker = tmp_path / "state" / "stub.done.json"
|
marker = tmp_path / "state" / "stub.done.json"
|
||||||
try:
|
try:
|
||||||
wait_for(marker.exists)
|
wait_for(marker.exists)
|
||||||
@@ -377,8 +290,7 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env:
|
|||||||
base = f"http://127.0.0.1:{port}"
|
base = f"http://127.0.0.1:{port}"
|
||||||
assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200
|
assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200
|
||||||
assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200
|
assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200
|
||||||
assert len(receiver.posts) == 1
|
assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": False, "summary": None}
|
||||||
assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": True, "summary": None}
|
|
||||||
proc.send_signal(signal.SIGTERM)
|
proc.send_signal(signal.SIGTERM)
|
||||||
assert proc.wait(timeout=10) == 0
|
assert proc.wait(timeout=10) == 0
|
||||||
finally:
|
finally:
|
||||||
@@ -386,9 +298,9 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env:
|
|||||||
proc.wait()
|
proc.wait()
|
||||||
|
|
||||||
|
|
||||||
def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
|
def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], tmp_path: Path) -> None:
|
||||||
port = free_port()
|
port = free_port()
|
||||||
proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="sleep", OUSAST_TERM_GRACE="1")
|
proc = spawn_runner(tmp_path, port, STUB_MODE="sleep", OUSAST_TERM_GRACE="1")
|
||||||
pids_file = tmp_path / "out" / "pids.json"
|
pids_file = tmp_path / "out" / "pids.json"
|
||||||
try:
|
try:
|
||||||
wait_for(pids_file.exists)
|
wait_for(pids_file.exists)
|
||||||
@@ -403,45 +315,21 @@ def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: di
|
|||||||
proc.kill()
|
proc.kill()
|
||||||
proc.wait()
|
proc.wait()
|
||||||
assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete"
|
assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete"
|
||||||
assert receiver.posts == [], "nothing is delivered for an interrupted command"
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_completion_marker_prevents_a_rerun_on_resume(
|
def test_the_completion_marker_prevents_a_rerun_on_resume(
|
||||||
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
) -> None:
|
) -> None:
|
||||||
counter = tmp_path / "runs.txt"
|
counter = tmp_path / "runs.txt"
|
||||||
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
|
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter))
|
||||||
assert code == 0 and len(counter.read_text().splitlines()) == 1 and len(receiver.posts) == 1
|
assert code == 0 and len(counter.read_text().splitlines()) == 1
|
||||||
(tmp_path / "ws" / "candidates.json").write_text("changed by the task\n")
|
(tmp_path / "ws" / "candidates.json").write_text("changed by the task\n")
|
||||||
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
|
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter))
|
||||||
assert code == 0
|
assert code == 0
|
||||||
assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work"
|
assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work"
|
||||||
assert len(receiver.posts) == 1, "a delivered result is not delivered again"
|
|
||||||
assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once"
|
assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once"
|
||||||
|
|
||||||
|
|
||||||
def test_a_failed_delivery_is_retried_alone_on_the_next_boot(
|
|
||||||
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
||||||
) -> None:
|
|
||||||
counter = tmp_path / "runs.txt"
|
|
||||||
broken = Receiver(500)
|
|
||||||
try:
|
|
||||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url, STUB_COUNTER=str(counter))
|
|
||||||
finally:
|
|
||||||
broken.stop()
|
|
||||||
assert code == 2
|
|
||||||
marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
|
|
||||||
assert marker["delivered"] is False and marker["status"] == "failed" and marker["exit"] == 0
|
|
||||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
|
|
||||||
assert code == 0
|
|
||||||
assert len(counter.read_text().splitlines()) == 1, "only the delivery is retried"
|
|
||||||
with tarfile.open(fileobj=io.BytesIO(receiver.posts[0][1]), mode="r:") as archive:
|
|
||||||
summary = archive.extractfile("summary.json")
|
|
||||||
assert summary is not None and json.loads(summary.read())["status"] == "done", "the task's own summary"
|
|
||||||
marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
|
|
||||||
assert marker["delivered"] is True and marker["status"] == "done"
|
|
||||||
|
|
||||||
|
|
||||||
# --- git workspaces from the case cache and by shallow clone ----------------------------------------------------------------
|
# --- git workspaces from the case cache and by shallow clone ----------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -590,14 +478,13 @@ def test_git_commits_annotation_round_trips_to_the_checkout(
|
|||||||
"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata: {name: rt}\nspec: {tasks: [{name: stub, task: stub}]}\n"
|
"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata: {name: rt}\nspec: {tasks: [{name: stub, task: stub}]}\n"
|
||||||
)
|
)
|
||||||
run, manifests = reconciler.load_run(manifest)
|
run, manifests = reconciler.load_run(manifest)
|
||||||
*rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests, "unused")
|
*rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests)
|
||||||
assert rest[0]["metadata"] == {"name": "case"}, "annotations are stripped before ax"
|
assert rest[0]["metadata"] == {"name": "case"}, "annotations are stripped before ax"
|
||||||
assert rest[0]["spec"]["git"] == [{"name": "code", "repo": repo.as_uri(), "branch": "main"}], "no commit for ax"
|
assert rest[0]["spec"]["git"] == [{"name": "code", "repo": repo.as_uri(), "branch": "main"}], "no commit for ax"
|
||||||
overrides = {"OUSAST_OUTPUT_DIR": str(output)}
|
overrides = {"OUSAST_OUTPUT_DIR": str(output)}
|
||||||
task_doc["spec"]["env"] = [
|
task_doc["spec"]["env"] = [
|
||||||
{"name": e["name"], "value": overrides.get(e["name"], e["value"])}
|
{"name": e["name"], "value": overrides.get(e["name"], e["value"])}
|
||||||
for e in task_doc["spec"]["env"]
|
for e in task_doc["spec"]["env"]
|
||||||
if e["name"] != "OUSAST_ARTIFACT_URL"
|
|
||||||
]
|
]
|
||||||
assert json.loads({e["name"]: e["value"] for e in task_doc["spec"]["env"]}["OUSAST_GIT_PINS"]) == {"case/code": pin}
|
assert json.loads({e["name"]: e["value"] for e in task_doc["spec"]["env"]}["OUSAST_GIT_PINS"]) == {"case/code": pin}
|
||||||
monkeypatch.setenv("AX_TASK_YAML", yaml.safe_dump(task_doc))
|
monkeypatch.setenv("AX_TASK_YAML", yaml.safe_dump(task_doc))
|
||||||
@@ -669,11 +556,11 @@ def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict
|
|||||||
|
|
||||||
|
|
||||||
def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child(
|
def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child(
|
||||||
env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
env: dict[str, str], tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||||
) -> None:
|
) -> None:
|
||||||
caplog.set_level(logging.DEBUG)
|
caplog.set_level(logging.DEBUG)
|
||||||
counter = tmp_path / "runs.txt"
|
counter = tmp_path / "runs.txt"
|
||||||
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
|
thread, environ, codes = main_in_thread(tmp_path, STUB_COUNTER=str(counter))
|
||||||
port = environ["AX_RUNNER_BOUND_PORT"]
|
port = environ["AX_RUNNER_BOUND_PORT"]
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request"
|
assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request"
|
||||||
@@ -693,15 +580,14 @@ def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only
|
|||||||
assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there"
|
assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there"
|
||||||
assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files"
|
assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files"
|
||||||
assert files_containing(tmp_path, SECRET) == [], "no file holds the credential"
|
assert files_containing(tmp_path, SECRET) == [], "no file holds the credential"
|
||||||
assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar"
|
|
||||||
assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record"
|
assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record"
|
||||||
assert any("start request: 202" in record.getMessage() for record in caplog.records)
|
assert any("start request: 202" in record.getMessage() for record in caplog.records)
|
||||||
|
|
||||||
|
|
||||||
def test_a_crash_that_prints_the_credential_is_redacted(
|
def test_a_crash_that_prints_the_credential_is_redacted(
|
||||||
env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str]
|
env: dict[str, str], tmp_path: Path, capfd: pytest.CaptureFixture[str]
|
||||||
) -> None:
|
) -> None:
|
||||||
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak")
|
thread, environ, codes = main_in_thread(tmp_path, STUB_MODE="leak")
|
||||||
start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
|
start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
|
||||||
assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202
|
assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202
|
||||||
thread.join(timeout=30)
|
thread.join(timeout=30)
|
||||||
@@ -727,23 +613,25 @@ def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch:
|
|||||||
assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists()
|
assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists()
|
||||||
|
|
||||||
|
|
||||||
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
|
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path) -> None:
|
||||||
"""Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed
|
"""Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed
|
||||||
actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start."""
|
actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start."""
|
||||||
counter = tmp_path / "runs.txt"
|
counter = tmp_path / "runs.txt"
|
||||||
same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"}
|
same = {"STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1", "AX_RUNNER_INSECURE_DELIVERY": "1"}
|
||||||
real_port, golden_port = free_port(), free_port()
|
real_port, golden_port = free_port(), free_port()
|
||||||
real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same)
|
real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same)
|
||||||
golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same)
|
golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same)
|
||||||
try:
|
try:
|
||||||
for port in (real_port, golden_port):
|
for port in (real_port, golden_port):
|
||||||
wait_for(lambda port=port: ready_on(port))
|
wait_for(lambda port=port: ready_on(port))
|
||||||
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET})
|
Router(f"http://127.0.0.1:{real_port}").start_task(
|
||||||
|
"default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}, {"put": object_store.presign_put("output.tar")}
|
||||||
|
)
|
||||||
marker = tmp_path / "state" / "stub.done.json"
|
marker = tmp_path / "state" / "stub.done.json"
|
||||||
wait_for(marker.exists)
|
wait_for(marker.exists)
|
||||||
time.sleep(1.0)
|
time.sleep(1.0)
|
||||||
assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command"
|
assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command"
|
||||||
assert golden.poll() is None and len(receiver.posts) == 1
|
assert golden.poll() is None
|
||||||
with pytest.raises(StartError, match="already started"):
|
with pytest.raises(StartError, match="already started"):
|
||||||
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {})
|
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {})
|
||||||
again_port = free_port()
|
again_port = free_port()
|
||||||
@@ -762,10 +650,10 @@ def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: R
|
|||||||
assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"
|
assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"
|
||||||
|
|
||||||
|
|
||||||
def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
|
def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], tmp_path: Path) -> None:
|
||||||
"""Substrate's golden boot never gets a start, so it must touch no workspace (and no network): a clone at boot
|
"""Substrate's golden boot never gets a start, so it must touch no workspace (and no network): a clone at boot
|
||||||
failed on the live cluster while the actor was still being restored (2026-09-29)."""
|
failed on the live cluster while the actor was still being restored (2026-09-29)."""
|
||||||
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url)
|
thread, environ, codes = main_in_thread(tmp_path)
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
written = [p for p in (tmp_path / "ws").rglob("*") if p.is_file()] if (tmp_path / "ws").exists() else []
|
written = [p for p in (tmp_path / "ws").rglob("*") if p.is_file()] if (tmp_path / "ws").exists() else []
|
||||||
assert written == [], "nothing is materialised before the start request"
|
assert written == [], "nothing is materialised before the start request"
|
||||||
@@ -778,82 +666,6 @@ def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str
|
|||||||
# --- inputs other tasks produced: fetched from the receiver after the start (Requirement 3.1, 4.4) -------------------
|
# --- inputs other tasks produced: fetched from the receiver after the start (Requirement 3.1, 4.4) -------------------
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def plane_receiver(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[egress.Receiver]:
|
|
||||||
"""The reconciler's own receiver on a free port, with a producer's artifact delivered and the task ``stub`` running."""
|
|
||||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
|
|
||||||
server = egress.Receiver("r", tmp_path / "run", 0)
|
|
||||||
(tmp_path / "run" / "repo-facts").mkdir(parents=True)
|
|
||||||
(tmp_path / "run" / "repo-facts" / "facts.json").write_bytes(b'{"a.py": ["f"]}' * 5000)
|
|
||||||
server.running.add("stub")
|
|
||||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
||||||
try:
|
|
||||||
yield server
|
|
||||||
finally:
|
|
||||||
server.shutdown()
|
|
||||||
server.server_close()
|
|
||||||
|
|
||||||
|
|
||||||
def input_env(server: egress.Receiver, tmp_path: Path) -> dict[str, str]:
|
|
||||||
return {
|
|
||||||
"OUSAST_ARTIFACT_URL": server.url,
|
|
||||||
"OUSAST_RUN": "r",
|
|
||||||
"OUSAST_TASK": "stub",
|
|
||||||
"OUSAST_INPUTS": json.dumps({"FACTS": "repo-facts/facts.json"}),
|
|
||||||
"OUSAST_INPUT_FACTS": str(tmp_path / "ws" / "in" / "repo-facts" / "facts.json"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_inputs_are_fetched_from_the_receiver_before_the_command(
|
|
||||||
env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
||||||
) -> None:
|
|
||||||
plane_receiver.inputs["stub"] = frozenset({"repo-facts/facts.json"})
|
|
||||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path))
|
|
||||||
assert code == 0
|
|
||||||
fetched = tmp_path / "ws" / "in" / "repo-facts" / "facts.json"
|
|
||||||
assert fetched.read_bytes() == (tmp_path / "run" / "repo-facts" / "facts.json").read_bytes()
|
|
||||||
assert "in/repo-facts/facts.json" in json.loads((output / "facts.json").read_text())["workspace_files"], "there before the command ran"
|
|
||||||
assert "stub" in plane_receiver.delivered
|
|
||||||
|
|
||||||
|
|
||||||
def test_an_input_that_cannot_be_fetched_delivers_a_failed_summary_and_never_runs_the_command(
|
|
||||||
env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
||||||
) -> None:
|
|
||||||
plane_receiver.inputs["stub"] = frozenset({"repo-facts/other.json"}) # facts.json is not among its inputs: 403
|
|
||||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path))
|
|
||||||
assert code == EXIT_BY_STATUS["failed"]
|
|
||||||
assert not (output / "facts.json").exists(), "the command never ran with an input missing"
|
|
||||||
delivered = json.loads((tmp_path / "run" / "stub" / "summary.json").read_text())
|
|
||||||
assert delivered["status"] == "failed" and "input FACTS (repo-facts/facts.json) not fetched: HTTP 403" in delivered["reason"]
|
|
||||||
assert not (tmp_path / "ws" / "in" / "repo-facts").exists() or not any((tmp_path / "ws" / "in" / "repo-facts").iterdir())
|
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_dials_the_override_and_sends_the_url_host(tmp_path: Path) -> None:
|
|
||||||
seen: list[dict[str, str]] = []
|
|
||||||
|
|
||||||
class _Handler(BaseHTTPRequestHandler):
|
|
||||||
def do_GET(self) -> None: # noqa: N802
|
|
||||||
seen.append({"path": self.path, **dict(self.headers.items())})
|
|
||||||
self.send_response(200)
|
|
||||||
self.send_header("Content-Length", "2")
|
|
||||||
self.end_headers()
|
|
||||||
self.wfile.write(b"ok")
|
|
||||||
|
|
||||||
def log_message(self, format: str, *args: object) -> None: # noqa: A002
|
|
||||||
pass
|
|
||||||
|
|
||||||
server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
|
||||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
||||||
try:
|
|
||||||
dial = f"127.0.0.1:{server.server_address[1]}"
|
|
||||||
size = runner.fetch("http://receiver.example/inputs/p/a.json", {"X-Ousast-Run": "r"}, tmp_path / "x" / "a.json", dial=dial)
|
|
||||||
finally:
|
|
||||||
server.shutdown()
|
|
||||||
server.server_close()
|
|
||||||
assert size == 2 and (tmp_path / "x" / "a.json").read_bytes() == b"ok"
|
|
||||||
assert seen[0]["path"] == "/inputs/p/a.json" and seen[0]["Host"] == "receiver.example" and seen[0]["X-Ousast-Run"] == "r"
|
|
||||||
|
|
||||||
|
|
||||||
# --- store delivery through the start body (plane-on-kubernetes 2.2, design section 3) ---------------------------
|
# --- store delivery through the start body (plane-on-kubernetes 2.2, design section 3) ---------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -905,7 +717,7 @@ def test_output_is_put_to_the_presigned_url_and_inputs_are_fetched_from_theirs(
|
|||||||
env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||||
) -> None:
|
) -> None:
|
||||||
"""End to end against the S3-shaped fake: the start carries the URLs, the runner PUTs the tar and GETs the input;
|
"""End to end against the S3-shaped fake: the start carries the URLs, the runner PUTs the tar and GETs the input;
|
||||||
no receiver, no OUSAST_ARTIFACT_URL, no URL in any log line."""
|
no URL in any log line."""
|
||||||
caplog.set_level(logging.DEBUG)
|
caplog.set_level(logging.DEBUG)
|
||||||
object_store._put("runs/run-1/repo-facts/facts.json", b'{"a.py": ["f"]}')
|
object_store._put("runs/run-1/repo-facts/facts.json", b'{"a.py": ["f"]}')
|
||||||
put_url = object_store.presign_put("runs/run-1/stub/output.tar")
|
put_url = object_store.presign_put("runs/run-1/stub/output.tar")
|
||||||
@@ -964,9 +776,9 @@ def test_a_failed_put_is_retried_alone_on_the_next_start_with_a_fresh_url(
|
|||||||
|
|
||||||
def test_deliver_puts_and_names_only_the_safe_url_on_failure(object_store: HttpObjectStore) -> None:
|
def test_deliver_puts_and_names_only_the_safe_url_on_failure(object_store: HttpObjectStore) -> None:
|
||||||
url = object_store.presign_put("k/output.tar")
|
url = object_store.presign_put("k/output.tar")
|
||||||
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, method="PUT") == 200
|
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0) == 200
|
||||||
assert object_store.objects["k/output.tar"] == b"tar"
|
assert object_store.objects["k/output.tar"] == b"tar"
|
||||||
with pytest.raises(runner.DeliveryError) as caught:
|
with pytest.raises(runner.DeliveryError) as caught:
|
||||||
runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0, method="PUT")
|
runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0)
|
||||||
assert "HTTP 404" in str(caught.value) and "X-Amz" not in str(caught.value) and "/other/k/output.tar" in str(caught.value)
|
assert "HTTP 404" in str(caught.value) and "X-Amz" not in str(caught.value) and "/other/k/output.tar" in str(caught.value)
|
||||||
assert runner.safe_url("https://h:1/p/q?X-Amz-Signature=s") == "https://h:1/p/q"
|
assert runner.safe_url("https://h:1/p/q?X-Amz-Signature=s") == "https://h:1/p/q"
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import subprocess
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from openultrasast.push_inputs import validate_manifest
|
from benchmarks.push.push_inputs import validate_manifest
|
||||||
|
|
||||||
|
|
||||||
def git(root, *args):
|
def git(root, *args):
|
||||||
@@ -118,7 +118,7 @@ def test_authored_edit_requires_exact_original_and_provenance(manifest):
|
|||||||
|
|
||||||
@pytest.mark.parametrize("field", ["snapshots", "cases"])
|
@pytest.mark.parametrize("field", ["snapshots", "cases"])
|
||||||
def test_empty_population_rejected(manifest, field):
|
def test_empty_population_rejected(manifest, field):
|
||||||
from openultrasast.push_inputs import InputError
|
from benchmarks.push.push_inputs import InputError
|
||||||
|
|
||||||
data, _, run = manifest
|
data, _, run = manifest
|
||||||
data[field] = []
|
data[field] = []
|
||||||
@@ -127,7 +127,7 @@ def test_empty_population_rejected(manifest, field):
|
|||||||
|
|
||||||
|
|
||||||
def test_missing_snapshot_identity_rejected(manifest):
|
def test_missing_snapshot_identity_rejected(manifest):
|
||||||
from openultrasast.push_inputs import InputError
|
from benchmarks.push.push_inputs import InputError
|
||||||
|
|
||||||
data, _, run = manifest
|
data, _, run = manifest
|
||||||
del data["snapshots"][0]["id"]
|
del data["snapshots"][0]["id"]
|
||||||
|
|||||||
+1
-21
@@ -5,14 +5,7 @@ import pytest
|
|||||||
|
|
||||||
from openultrasast.preprocess import FileTarget
|
from openultrasast.preprocess import FileTarget
|
||||||
from openultrasast.provider.openrouter import OpenRouterError, parse_json_content
|
from openultrasast.provider.openrouter import OpenRouterError, parse_json_content
|
||||||
from openultrasast.rank import (
|
from openultrasast.rank import RankingError, composite_priority, parse_ranker_response, rank_targets, write_rankings
|
||||||
RankingError,
|
|
||||||
composite_priority,
|
|
||||||
parse_ranker_response,
|
|
||||||
rank_targets,
|
|
||||||
rank_targets_with_model,
|
|
||||||
write_rankings,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeRankerClient:
|
class FakeRankerClient:
|
||||||
@@ -72,19 +65,6 @@ def test_parse_ranker_response_rejects_malformed_scores() -> None:
|
|||||||
parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model")
|
parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model")
|
||||||
|
|
||||||
|
|
||||||
def test_rank_targets_with_model_chunks_requests() -> None:
|
|
||||||
targets = [
|
|
||||||
FileTarget(path="a.py", absolute_path="/repo/a.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
|
|
||||||
FileTarget(path="b.py", absolute_path="/repo/b.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
|
|
||||||
]
|
|
||||||
client = FakeRankerClient()
|
|
||||||
|
|
||||||
rankings = rank_targets_with_model(targets, client=client, model="openrouter/test-ranker", chunk_size=1)
|
|
||||||
|
|
||||||
assert client.calls == 2
|
|
||||||
assert {ranking.model_id for ranking in rankings} == {"openrouter/test-ranker"}
|
|
||||||
|
|
||||||
|
|
||||||
def test_parse_json_content_accepts_fenced_json() -> None:
|
def test_parse_json_content_accepts_fenced_json() -> None:
|
||||||
assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []}
|
assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []}
|
||||||
|
|
||||||
|
|||||||
@@ -169,33 +169,6 @@ def test_claim_is_visible_in_worker_snapshot(tmp_path):
|
|||||||
Coordinator(board(tmp_path), execute, budget=SearchBudget(reason_rounds=1)).run()
|
Coordinator(board(tmp_path), execute, budget=SearchBudget(reason_rounds=1)).run()
|
||||||
|
|
||||||
|
|
||||||
def test_batch_adapter_offline_checkpoint_and_step_dispatch(tmp_path, monkeypatch):
|
|
||||||
from dataclasses import asdict
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1]))
|
|
||||||
from openultrasast.search.coordinator import BatchExecutor, SearchTask
|
|
||||||
|
|
||||||
calls = []
|
|
||||||
|
|
||||||
def lane(item, output, attempt):
|
|
||||||
import json
|
|
||||||
|
|
||||||
spec = json.loads(item.inputs["SPEC_URL"])
|
|
||||||
calls.append(spec)
|
|
||||||
assert item.extra_env["SEARCH_STEP"] == spec["step"]
|
|
||||||
return {"status": "ok", "cost_usd": 0, "intents": []}
|
|
||||||
|
|
||||||
adapter = BatchExecutor(
|
|
||||||
image="registry/task@sha256:" + "a" * 64, command=("python3", "/worker.py"), output=tmp_path, lanes={"ax": lane}
|
|
||||||
)
|
|
||||||
for index, step in enumerate(("reason", "explore", "verify")):
|
|
||||||
task = SearchTask(f"task-{index}", "search-1", step, "facts: []", {}, asdict(SearchBudget()))
|
|
||||||
assert adapter(task)["status"] == "ok"
|
|
||||||
assert adapter(task)["status"] == "ok"
|
|
||||||
assert len(calls) == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_invalid_identifiers_refused_before_paths(tmp_path):
|
def test_invalid_identifiers_refused_before_paths(tmp_path):
|
||||||
from dataclasses import asdict
|
from dataclasses import asdict
|
||||||
|
|
||||||
|
|||||||
@@ -10,22 +10,11 @@ from openultrasast.verification import (
|
|||||||
VerificationStatus,
|
VerificationStatus,
|
||||||
build_independent_verifier_context,
|
build_independent_verifier_context,
|
||||||
is_report_verified,
|
is_report_verified,
|
||||||
validate_evidence_transition,
|
|
||||||
verify_finding,
|
verify_finding,
|
||||||
write_verification_results,
|
write_verification_results,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_evidence_transition_rejects_backward_and_skipped_levels() -> None:
|
|
||||||
assert validate_evidence_transition("suspicion", "static_corroboration") == EvidenceLevel.STATIC_CORROBORATION
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="backward"):
|
|
||||||
validate_evidence_transition("root_cause_explained", "static_corroboration")
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="skip"):
|
|
||||||
validate_evidence_transition("suspicion", "root_cause_explained")
|
|
||||||
|
|
||||||
|
|
||||||
def test_verified_status_requires_static_corroboration_and_acceptance() -> None:
|
def test_verified_status_requires_static_corroboration_and_acceptance() -> None:
|
||||||
assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED)
|
assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED)
|
||||||
assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED)
|
assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED)
|
||||||
|
|||||||
Reference in New Issue
Block a user