marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 11:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery

Browse Source 
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts).
Deleted: the four hunter tools no caller enables (read_definition,
entry_points, flows, obligations) with their schemas and dispatch, the dead
mapping branch of find_refs; 20 functions whose only caller was one test
(mapping task records, compute_ranking_metrics, build_quick_hunter_prompt,
validate_evidence_transition, should_fuse, harness processors,
rank_targets_with_model, model/report slice rendering, taxonomy
hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles,
features.language_of and build_for_scan, folds.leave_one_*,
features.recorded_case, search BatchExecutor and the three __main__ shims,
delivery.put_input); the receiver delivery mode of the plane. push_inputs.py
moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses
it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282
tests collect. breadth.toml records the removals.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-13 11:06:09 +02:00
co-authored by Claude Fable 5.1
parent f5a904b7f2
commit f63703a604
60 changed files with 212 additions and 2576 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_calibration.py
-29
View File
@@ -6,7 +6,6 @@ from openultrasast.calibration import (
apply_ranking_calibration,
calibrate_ranking,
calibrate_rankings,
compute_ranking_metrics,
learning_from_finding_outcome,
record_false_positive_learning,
write_false_positive_learnings,
@@ -62,34 +61,6 @@ def test_false_positive_learning_records_reason_and_scope(tmp_path: Path) -> Non
assert "missing_attacker_control" in output.read_text()
def test_ranking_metrics_track_false_positive_reasons_by_tier() -> None:
metrics = compute_ranking_metrics(
[
{"tier": "A", "outcome": FindingOutcome.VERIFIED, "time_to_verification_seconds": 10},
{
"tier": "A",
"outcome": FindingOutcome.REJECTED,
"reason": FalsePositiveReason.UNREACHABLE_PATH,
"time_to_verification_seconds": 20,
},
{"tier": "B", "outcome": FindingOutcome.REJECTED, "reason": FalsePositiveReason.DUPLICATE},
{
"tier": "A",
"outcome": FindingOutcome.UNVERIFIED,
"reason": FalsePositiveReason.UNVERIFIED,
"missed_fixture_vulnerabilities": 1,
},
],
tier="A",
)
assert metrics.verified_findings == 1
assert metrics.false_positives == 2
assert metrics.time_to_verification_seconds == 15.0
assert metrics.missed_fixture_vulnerabilities == 1
assert metrics.false_positive_reasons["unreachable_path"] == 1
def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None:
learning = record_false_positive_learning(
_finding("auth/login.py"),
tests/test_findings.py
+1 -15
View File
@@ -1,7 +1,7 @@
import json
from pathlib import Path
from openultrasast.findings import build_quick_hunter_prompt, quick_scan_findings, write_findings
from openultrasast.findings import quick_scan_findings, write_findings
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
@@ -92,17 +92,3 @@ def test_findings_inherit_only_matching_function_level_reachability(tmp_path: Pa
assert unreachable.ranking_priority <= 1.5
def test_quick_hunter_prompt_is_bounded() -> None:
repo = Path("/repo/example.py")
target = next(iter(preprocess_repository(repo.parent)[1]), None) if repo.parent.exists() else None
assert target is None
class Target:
path = "example.py"
language = "python"
tags = ["auth_boundary"]
prompt = build_quick_hunter_prompt(Target(), "x" * 5000) # type: ignore[arg-type]
assert "example.py" in prompt
assert len(prompt) < 4300
tests/test_fusion.py
+1 -15
View File
@@ -6,13 +6,7 @@ from pathlib import Path
import pytest
from openultrasast.findings import StaticFinding
from openultrasast.fusion import (
FusionDisposition,
fuse_finding,
fuse_findings,
fusion_triggers,
should_fuse,
)
from openultrasast.fusion import FusionDisposition, fuse_finding, fuse_findings, fusion_triggers
from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus
@@ -63,10 +57,6 @@ def test_triggers_on_high_severity_and_reachable() -> None:
assert "gates_risky_fix" in triggers
def test_low_severity_unreachable_does_not_trigger() -> None:
assert not should_fuse(_finding(severity="low", reachability="unknown"))
def test_verifier_disagreement_triggers() -> None:
triggers = fusion_triggers(
_finding(severity="low", reachability="unknown"),
@@ -83,10 +73,6 @@ def test_evidence_conflict_triggers_when_rejected_but_reachable() -> None:
assert "evidence_conflict" in triggers
def test_high_assurance_request_forces_a_trigger() -> None:
assert should_fuse(_finding(severity="low", reachability="unknown"), high_assurance=True)
# ---- deterministic two-panel decider + dispositions -------------------------
tests/test_harness.py
+4 -68
View File
@@ -1,100 +1,36 @@
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import pytest
from openultrasast.config import load_config
from openultrasast.harness import (
HarnessContractError,
HarnessEvent,
HarnessRuntime,
HarnessTraceWriter,
ProcessorSpec,
write_harness_config,
)
from openultrasast.harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
@dataclass
class RecordingProcessor:
spec: ProcessorSpec
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
return {"seen_stage": event.stage}
@dataclass
class BadWriteProcessor:
spec: ProcessorSpec
def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
return {"undeclared": event.event_type}
def test_runtime_emits_stage_events_and_processor_state(tmp_path: Path) -> None:
def test_runtime_emits_stage_events(tmp_path: Path) -> None:
trace_path = tmp_path / "events.jsonl"
processor = RecordingProcessor(ProcessorSpec(name="recorder", version="1", handles=("stage_start",), writes=("seen_stage",)))
runtime = HarnessRuntime(
scan_id="scan-1",
config=load_config(None),
trace_writer=HarnessTraceWriter(trace_path),
processors=[processor],
)
result = runtime.run_stage("preprocess", lambda: "ok")
events = [json.loads(line) for line in trace_path.read_text().splitlines()]
assert result == "ok"
assert runtime.state["seen_stage"] == "preprocess"
assert [event["event_type"] for event in events] == ["stage_start", "stage_start", "stage_end"]
assert events[1]["payload"] == {"changed_state": ["seen_stage"], "processor": "recorder"}
assert [event["event_type"] for event in events] == ["stage_start", "stage_end"]
def test_strict_contract_mode_rejects_undeclared_writes(tmp_path: Path) -> None:
processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
runtime = HarnessRuntime(
scan_id="scan-1",
config=load_config(None),
trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
processors=[processor],
)
with pytest.raises(HarnessContractError, match="undeclared writes"):
runtime.emit("stage_start", "rank", {})
def test_warn_contract_mode_records_degradation(tmp_path: Path) -> None:
processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
runtime = HarnessRuntime(
scan_id="scan-1",
config=load_config(None),
trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
processors=[processor],
contract_mode="warn",
)
runtime.emit("stage_start", "rank", {})
assert "undeclared writes" in runtime.state["degradations"][0]
assert '"status": "degraded"' in (tmp_path / "events.jsonl").read_text()
def test_harness_config_serializes_models_processors_and_prompt_hashes(tmp_path: Path) -> None:
def test_harness_config_serializes_models_and_prompt_hashes(tmp_path: Path) -> None:
config_path = tmp_path / "openultrasast.toml"
config_path.write_text('[models]\nranker = "openrouter/ranker"\n')
processor = RecordingProcessor(ProcessorSpec(name="recorder", version="2026-01-01", handles=("stage_start",), writes=("seen_stage",)))
output = tmp_path / "harness.json"
write_harness_config(
config=load_config(config_path),
processors=[processor],
contract_mode="strict",
path=output,
prompts={"ranker": "score files"},
)
payload = json.loads(output.read_text())
assert payload["model_roles"]["ranker"] == "openrouter/ranker"
assert payload["processor_versions"] == {"recorder": "2026-01-01"}
assert len(payload["prompt_hashes"]["ranker"]) == 64
tests/test_hunter_curated_tools.py
-198
View File
@@ -1,198 +0,0 @@
"""learning-harness task 1.5: curated repository tools and hunter-loop parameters (offline)."""
from __future__ import annotations
import json
import tempfile
from pathlib import Path
import pytest
from openultrasast.hunter_tools import PathEscapesRepo
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter
APP = (
"from flask import request\n\n\n"
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
" return Book.query.filter_by(book_title=title).first()\n\n\n"
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
" owner = request.args.get('owner')\n"
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n"
"def helper(value):\n"
" return value.strip()\n"
)
@pytest.fixture
def repo(tmp_path: Path) -> Path:
(tmp_path / "app.py").write_text(APP)
return tmp_path
def test_read_definition_returns_one_span_and_refuses_to_leave_the_repository(repo: Path) -> None:
from openultrasast.hunter_tools import read_definition
found = read_definition(repo, "leaky", max_chars=4000)
assert found is not None
assert found["path"] == "app.py" and found["name"] == "leaky"
assert found["start_line"] == 10 and found["end_line"] == 13
assert str(found["text"]).startswith("@app.route('/books/any/<title>')") # a handler's decorators come with it
assert "def leaky" in str(found["text"]) and "def guarded" not in str(found["text"])
assert read_definition(repo, "not_a_function", max_chars=4000) is None
with pytest.raises(PathEscapesRepo):
read_definition(repo, "leaky", max_chars=4000, path="../outside.py")
def test_entry_points_lists_handlers_with_their_access_and_never_source_text(repo: Path) -> None:
from openultrasast.hunter_tools import entry_points
rows = entry_points(repo)
by_function = {str(row["function"]): row for row in rows if row.get("function")}
assert {"guarded", "leaky"} <= set(by_function)
assert by_function["guarded"]["access_level"] == "authenticated" and by_function["leaky"]["access_level"] == "public"
assert by_function["leaky"]["path"] == "app.py" and isinstance(by_function["leaky"]["line"], int)
assert all("text" not in row and "content" not in row for row in rows) # identifiers and spans only
assert entry_points(repo, path="app.py") == rows
with pytest.raises(PathEscapesRepo):
entry_points(repo, path="../elsewhere.py")
def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path) -> None:
from openultrasast.hunter_tools import obligations
rows = obligations(repo, path="app.py", function="leaky")
assert rows and all(row["function"] == "leaky" for row in rows)
assert {"protected_read"} == {str(row["operation"]) for row in rows}
assert "identity_constraint" in {str(row["missing"]) for row in rows}
assert all("text" not in row and "content" not in row for row in rows)
assert obligations(repo, path="app.py", function="helper") == []
with pytest.raises(PathEscapesRepo):
obligations(repo, path="../outside.py", function="leaky")
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
from openultrasast.hunter_tools import flows
(repo / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n")
rows = flows(repo, path="flow.py", function="run")
assert all(row["function"] == "run" and row["path"] == "flow.py" for row in rows)
assert all(set(row) <= {"path", "function", "line", "sources", "sinks", "disposition", "cwe"} for row in rows)
assert flows(repo, path="app.py", function="helper") == []
with pytest.raises(PathEscapesRepo):
flows(repo, path="/etc/passwd", function="run")
def test_the_loop_keeps_todays_behaviour_by_default(repo: Path) -> None:
from openultrasast.complexity.map import Hotspot
from openultrasast.tool_hunter import DEFAULT_TOOL_NAMES
hotspot = Hotspot(
path="app.py",
function_name=None,
score=1.0,
band="high",
signals={},
rationale="fixture",
test_hint=None,
inventory_finding_ids=(),
)
client = ScriptedChatClient(
[
ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)),
ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'),
]
)
findings = run_tool_hunter(repo, [hotspot], client=client, model="m", max_steps=4)
assert [finding.finding_id for finding in findings] == ["tool-hunter:app.py:12"]
assert findings[0].tags == ["tool-hunter"] and findings[0].evidence_level == "suspicion"
assert tuple(tool["function"]["name"] for tool in client.calls[0]["tools"]) == DEFAULT_TOOL_NAMES
def test_the_loop_takes_a_prompt_context_tools_and_tags_and_replays_reasoning(repo: Path) -> None:
from openultrasast.complexity.map import Hotspot
hotspot = Hotspot(
path="app.py",
function_name="leaky",
score=1.0,
band="high",
signals={},
rationale="fixture",
test_hint=None,
inventory_finding_ids=(),
)
client = ScriptedChatClient(
[
ChatResponse(
content="looking",
reasoning="first I check the route",
tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),),
),
ChatResponse(content='[{"path": "app.py", "line": 12, "title": "t", "rationale": "r"}]'),
]
)
findings = run_tool_hunter(
repo,
[hotspot],
client=client,
model="m",
max_steps=4,
system_prompt="You hunt access control only.",
user_prompt="Look at leaky.",
context_files=("app.py",),
tools=("read_definition", "obligations"),
tags=("family:access_control", "detector:access_control@1"),
)
first = client.calls[0]
assert first["messages"][0]["content"] == "You hunt access control only."
assert "Look at leaky." in str(first["messages"][1]["content"]) and "def leaky" in str(first["messages"][1]["content"])
assert [tool["function"]["name"] for tool in first["tools"]] == ["read_definition", "obligations"]
replayed = [message for message in client.calls[1]["messages"] if message.get("role") == "assistant"]
assert replayed and replayed[0]["reasoning_content"] == "first I check the route"
tool_reply = json.loads(str(client.calls[1]["messages"][-1]["content"]))
assert tool_reply and tool_reply[0]["missing"] == "identity_constraint"
assert findings[0].tags == ["family:access_control", "detector:access_control@1"]
def test_the_loop_asks_once_more_for_the_answer_instead_of_dropping_prose() -> None:
"""Measured live: the model spends its step budget on tools and then answers in prose, and the whole hunt is
discarded (learning-harness Req 6.3).
Four steps, three of them tool calls, and the fourth a paragraph that names the sink, the argument and the
line — parsed as nothing. A detector that found the bug and reported zero is indistinguishable from one that
found nothing, and every family in round zero would have baselined at zero for this reason alone."""
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall, run_tool_hunter
prose = "The function passes `command` straight to os.system at line 2, an OS command sink."
answer = json.dumps([{"path": "app.py", "line": 2, "title": "command injection", "rationale": "os.system(command)"}])
client = ScriptedChatClient(
[
ChatResponse(tool_calls=(ToolCall(id="c1", name="grep_repo", arguments={"pattern": "system"}),)),
ChatResponse(tool_calls=(ToolCall(id="c2", name="read_file", arguments={"path": "app.py"}),)),
ChatResponse(content=prose),
ChatResponse(content=answer), # the one extra turn, asked for explicitly
]
)
with tempfile.TemporaryDirectory() as scratch:
root = Path(scratch)
(root / "app.py").write_text("import os\ndef run(command):\n return os.system(command)\n")
findings = run_tool_hunter(root, [], client=client, model="m", max_steps=3)
assert [item.line for item in findings] == [2]
final = client.calls[-1]
# Asked for JSON in the prompt, not through the provider's JSON mode: with tools attached, DeepSeek's JSON mode
# returned `{"type": "json_object"}` and lost the answers (tests/test_tool_hunter.py pins the shapes).
assert not final.get("json_object") and final.get("tools") == [], "the last turn is plain text without tools"
text = "\n".join(str(message.get("content") or "") for message in final["messages"])
assert "JSON" in text and prose in text, "the prose it already wrote is what the final turn is asked to format"
def test_a_hunt_that_never_called_a_tool_still_reports_nothing() -> None:
"""The extra turn is for formatting an answer, never for inventing one."""
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, run_tool_hunter
client = ScriptedChatClient([ChatResponse(content="I think there is an injection here."), ChatResponse(content="[]")])
with tempfile.TemporaryDirectory() as scratch:
root = Path(scratch)
(root / "app.py").write_text("x = 1\n")
assert run_tool_hunter(root, [], client=client, model="m", max_steps=3) == []
assert len(client.calls) == 1, "no tool was called, so there is nothing to format"
tests/test_hunter_tools.py
+6 -36
View File
@@ -3,7 +3,6 @@ from pathlib import Path
import pytest
from openultrasast.hunter_tools import PathEscapesRepo, clamp_repo_path, find_refs, grep_repo, read_file
from openultrasast.mapping import EntryPointRecord
SPLIT_SINK = """\
term = request.args["q"]
@@ -85,42 +84,13 @@ def test_read_file_truncates_and_rejects_symlink_escape(tmp_path: Path) -> None:
read_file(root, "link.py", max_chars=100)
def test_find_refs_uses_mapping_index_and_clamped_text_search(tmp_path: Path) -> None:
def test_find_refs_searches_source_text(tmp_path: Path) -> None:
root = _repo_with_split_sink(tmp_path)
mapping_index = [
{"path": "app.py", "name": "query", "line": 2},
EntryPointRecord(
path="app.py",
line=3,
end_line=3,
function_name="search",
name="search",
kind="route",
access_level="public",
trust_boundary="http_request",
access_evidence=[],
conditions=[],
provenance="test",
rationale="fixture",
),
]
query_refs = find_refs(root, "query", mapping_index)
assert any(ref["path"] == "app.py" and ref["name"] == "query" for ref in query_refs)
assert any(ref.get("source") == "mapping" for ref in query_refs)
assert any(ref.get("source") == "text" and "query" in str(ref.get("text", ref.get("name", ""))) for ref in query_refs)
search_refs = find_refs(root, "search", mapping_index)
assert any(ref["path"] == "app.py" and ref["name"] == "search" for ref in search_refs)
def test_find_refs_rejects_mapping_path_escape(tmp_path: Path) -> None:
root = _repo_with_split_sink(tmp_path)
with pytest.raises(PathEscapesRepo):
find_refs(root, "passwd", [{"path": "../etc/passwd", "name": "passwd"}])
with pytest.raises(PathEscapesRepo):
find_refs(root, "passwd", [{"path": "/etc/passwd", "name": "passwd"}])
refs = find_refs(root, "query")
assert [ref["line"] for ref in refs] == [2, 3]
assert all(ref["path"] == "app.py" and ref["name"] == "query" and ref["source"] == "text" for ref in refs)
assert find_refs(root, "search") == []
assert find_refs(root, "") == []
def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> None:
tests/test_learn_folds.py
+1 -26
View File
@@ -5,17 +5,7 @@ from __future__ import annotations
import pytest
from learn_fixtures import corpus
from openultrasast.learn.folds import (
INSUFFICIENT,
Fold,
FoldError,
assert_disjoint,
compile_split,
folds_digest,
leave_one_framework_out,
leave_one_source_out,
outer_folds,
)
from openultrasast.learn.folds import Fold, FoldError, assert_disjoint, compile_split, folds_digest, outer_folds
def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None:
@@ -26,8 +16,6 @@ def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation
assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4
folds = [
*outer_folds(examples, seed=1, exclude=split.groups),
*leave_one_source_out(examples, exclude=split.groups),
*leave_one_framework_out(examples, exclude=split.groups, min_groups=2),
]
assert_disjoint(split, folds)
for fold in folds:
@@ -52,19 +40,6 @@ def test_folds_are_deterministic_by_seed() -> None:
assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6))
def test_leave_one_source_and_framework_out() -> None:
examples = corpus(groups=12)
sources = leave_one_source_out(examples)
assert {f.held_source for f in sources} == {"pairs", "population-v1", "population-v2"}
for fold in sources:
assert fold.eval_groups == {e.group for e in examples if e.source == fold.held_source}
frameworks = {f.held_framework: f for f in leave_one_framework_out(examples)}
assert set(frameworks) == {"flask", "django"}
assert all(f.status == INSUFFICIENT and not f.eval_groups for f in frameworks.values()) # 3 groups each < 10
enough = {f.held_framework: f for f in leave_one_framework_out(examples, min_groups=3)}
assert enough["flask"].status == "ok" and enough["flask"].eval_groups == {e.group for e in examples if "flask" in e.frameworks}
def test_a_shared_group_is_refused() -> None:
examples = corpus(groups=8)
split = compile_split(examples, seed=0)
tests/test_learn_retrieve.py
+3 -18
View File
@@ -8,7 +8,7 @@ from dataclasses import replace
import pytest
from learn_fixtures import by_id, corpus, example, x_record
from openultrasast.learn.folds import Fold, compile_split, leave_one_framework_out, leave_one_source_out, outer_folds
from openultrasast.learn.folds import Fold, compile_split, outer_folds
from openultrasast.learn.retrieve import (
BoundaryViolation,
Target,
@@ -166,13 +166,11 @@ def _all_folds(examples: list) -> list:
split = compile_split(examples, seed=3)
return [
*outer_folds(examples, seed=3, exclude=split.groups),
*leave_one_source_out(examples, exclude=split.groups),
*[f for f in leave_one_framework_out(examples, exclude=split.groups, min_groups=2) if f.status == "ok"],
]
def test_boundary() -> None:
"""For outer, leave-one-source-out and leave-one-framework-out folds, every retrieved example and every
"""For outer folds, every retrieved example and every
demonstration that reaches a prompt is eligible; ``seed_memory`` writes no example of the case's group."""
examples = corpus(groups=16)
index = by_id(examples)
@@ -186,7 +184,7 @@ def test_boundary() -> None:
)
]
folds = _all_folds(examples)
assert {f.kind for f in folds} == {"outer", "leave_one_source_out", "leave_one_framework_out"}
assert {f.kind for f in folds} == {"outer"}
checked = 0
for fold in folds:
for target_example in [e for e in examples if e.group in fold.eval_groups]:
@@ -206,19 +204,6 @@ def test_boundary() -> None:
assert all(e.group != group for e in seed_memory(examples, group))
def test_a_demonstration_of_the_held_out_framework_is_swapped_for_its_alternate() -> None:
flask = example("demo-flask", group="d/flask", label=1, frameworks=("flask",))
alt = example("demo-plain", group="d/plain", label=1)
target_example = example("t", group="t/t", label=0, frameworks=("flask",))
target = Target.of(target_example)
fold = leave_one_framework_out([flask, target_example], min_groups=1)[0]
assert fold.held_framework == "flask" and target_example.group in fold.eval_groups
demos = [{"example_id": flask.id, "alternate": {"example_id": alt.id}}]
usable = demonstrations_for(demos, by_id([flask, alt]), target, fold)
assert [d["example_id"] for d in usable] == [alt.id]
assert demonstrations_for([{"example_id": flask.id, "alternate": None}], by_id([flask]), target, fold) == []
def test_assert_boundary_catches_an_ineligible_rendered_id() -> None:
examples = corpus(groups=6)
fold = outer_folds(examples, k=3)[0]
tests/test_learn_roles.py
+1 -17
View File
@@ -7,7 +7,7 @@ from __future__ import annotations
import json
from pathlib import Path
from openultrasast.learn.features import Vocabulary, build_for_scan, function_of, model_sinks_part, roles_part
from openultrasast.learn.features import Vocabulary, function_of, model_sinks_part, roles_part
from openultrasast.learn.roles import (
RoleSet,
base_roles,
@@ -168,22 +168,6 @@ def test_a_language_without_declarations_is_none_not_failed() -> None:
assert source_part(["def g():", " pass"], "python", "missing").state == "failed"
def test_build_for_scan_records_the_roles_instrument(tmp_path: Path) -> None:
from openultrasast.findings import quick_scan_findings
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
_write(tmp_path, "app/one.py", "import os\n\n\ndef inner(value):\n os.system(value)\n")
_, targets = preprocess_repository(tmp_path)
assert targets and (tmp_path / targets[0].path).stat().st_size > 0, "the fixture was read"
findings = quick_scan_findings(tmp_path, targets, rank_targets(targets))
records = build_for_scan(tmp_path, findings, None)
assert records and records[0]["instruments"]["roles"]["state"] == "ran"
assert records[0]["x"]["roles.sink_confidence"] == 1.0
off = build_for_scan(tmp_path, findings, None, roles=False)
assert off[0]["instruments"]["roles"]["state"] == "none" and off[0]["x"]["roles.sink_confidence"] is None
# --- model roles: the plane task under a scripted client ----------------------------------------------------------
tests/test_learn_schema.py
+1 -57
View File
@@ -10,14 +10,10 @@ from typing import Any
import pytest
from openultrasast.findings import quick_scan_findings
from openultrasast.learn import schema
from openultrasast.learn.features import Delta, Part, build_for_scan, record
from openultrasast.learn.features import Part, record
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
from openultrasast.plane.tasks.features import case_features
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
ROOT = Path(__file__).resolve().parents[1]
APP = """import os
@@ -32,22 +28,6 @@ def {outer}():
"""
def _scan(root: Path) -> list[dict[str, Any]]:
_, targets = preprocess_repository(root)
assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read"
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
findings = quick_scan_findings(root, targets, rank_targets(targets))
assert findings, "the fixture's os.system fires a quick rule"
engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []}
return build_for_scan(root, findings, engine)
def _repo(root: Path, path: str, inner: str, outer: str) -> Path:
(root / path).parent.mkdir(parents=True, exist_ok=True)
(root / path).write_text(APP.format(inner=inner, outer=outer))
return root
# --- the allow-list ----------------------------------------------------------------------------------------------
@@ -63,35 +43,10 @@ def test_features_are_allow_listed(tmp_path: Path) -> None:
mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta"))
assert records
for rec in records:
assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES"
assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"])
assert {s.name for s in schema.features_for("static")} < set(names)
assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]
def test_record_is_name_invariant(tmp_path: Path) -> None:
"""Two repositories that differ only in repository name, paths, file and function names and commit ids give the
same features: nothing identifies a repository."""
first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta"))
second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta"))
assert [r["x"] for r in first] == [r["x"] for r in second]
assert [r["instruments"] for r in first] == [r["instruments"] for r in second]
assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope"
assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None
root = tmp_path / "acme-shop"
_, targets = preprocess_repository(root)
findings = quick_scan_findings(root, targets, rank_targets(targets))
changed = {"app/one.py": [(5, 5)]}
one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40))
two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40))
assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40
assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1
assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature"
# --- label fields ------------------------------------------------------------------------------------------------
AGREED = {
@@ -165,17 +120,6 @@ def test_missing_is_not_zero() -> None:
assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"
def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None:
root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta")
_, targets = preprocess_repository(root)
findings = quick_scan_findings(root, targets, rank_targets(targets))
unread = build_for_scan(root, findings, {"questions": 0, "findings": []})
absent = build_for_scan(root, findings, None)
assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"}
assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"}
assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent))
def test_validate_record_refuses_values_outside_their_type() -> None:
good = _plane()[0]
validate_record(good)
tests/test_mapping.py
-56
View File
@@ -4,13 +4,7 @@ from pathlib import Path
from openultrasast.mapping import (
analyze_entry_points,
attach_reachability_hints,
attach_static_hints,
codeql_mapping_tasks,
differential_mapping_record,
ingest_sarif,
semgrep_mapping_tasks,
sharp_edge_record,
verifier_evidence_candidates,
write_entry_points,
write_static_hints,
)
@@ -32,56 +26,6 @@ def test_ingest_sarif_normalizes_semgrep_hints(tmp_path: Path) -> None:
assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval"
def test_static_hints_attach_to_file_targets_and_evidence_candidates(tmp_path: Path) -> None:
repo = tmp_path / "repo"
repo.mkdir()
(repo / "app.py").write_text("eval(user_input)\n")
hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "py/code-injection", "app.py"))
_, targets = preprocess_repository(repo, static_hints=hints)
enriched = attach_static_hints(targets, hints)
evidence = verifier_evidence_candidates(hints, path="app.py")
assert enriched[0].static_hints[0]["analyzer"] == "codeql"
assert evidence == [
{
"source": "codeql",
"rule_id": "py/code-injection",
"path": "app.py",
"line": 7,
"severity": "high",
"message": "Potential injection",
"provenance": "sarif:codeql:py/code-injection",
}
]
def test_mapping_task_records_cover_disciplines(tmp_path: Path) -> None:
semgrep_hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "parser.c"))
codeql_hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "cpp/overflow", "parser.c"))
assert semgrep_mapping_tasks(semgrep_hints)[0].task == "pattern_variant_review"
assert codeql_mapping_tasks(codeql_hints)[0].task == "source_sink_sanitizer_path_review"
assert (
differential_mapping_record(
"parser.c",
change_kind="modified_function",
trust_boundary="network_input",
rationale="Parser changed behind a socket boundary.",
).trust_boundary
== "network_input"
)
assert (
sharp_edge_record(
"config.py",
category="insecure_default",
surface="debug mode",
rationale="Debug default changes production risk.",
).category
== "insecure_default"
)
def test_write_static_hints_artifact(tmp_path: Path) -> None:
hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py"))
output = tmp_path / "mapping" / "static_hints.json"
tests/test_model_config_value.py
+24 -24
View File
@@ -26,17 +26,17 @@ def _cpg(rows): # type: ignore[no-untyped-def]
def test_a_permissive_literal_is_entailed() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}]
answer = verdict(_cpg(rows), _spec(), function="create_app")
answer = verdicts(_cpg(rows), _spec(), function="create_app")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
assert "*" in answer.witness and "CORS" in answer.witness
def test_a_restrictive_literal_yields_no_finding() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
rows = [
{
@@ -47,12 +47,12 @@ def test_a_restrictive_literal_yields_no_finding() -> None:
"args": ["app", '"https://app.example"'],
}
]
assert verdict(_cpg(rows), _spec(), function="create_app") is None
assert verdicts(_cpg(rows), _spec(), function="create_app") == []
def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
"""The value is computed, so constant abstraction cannot decide it — that residual is the LLM's."""
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung
rows = [
@@ -64,30 +64,30 @@ def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
"args": ["app", "configured_origins"],
}
]
answer = verdict(_cpg(rows), _spec(), function="create_app")
answer = verdicts(_cpg(rows), _spec(), function="create_app")[0]
assert answer is not None and answer.rung is Rung.CORROBORATED
def test_a_setting_that_is_not_in_the_spec_is_ignored() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
rows = [{"setting": 'print("*")', "line": "1", "method": "create_app", "literalArgs": ['"*"'], "args": ['"*"']}]
assert verdict(_cpg(rows), _spec(), function="create_app") is None
assert verdicts(_cpg(rows), _spec(), function="create_app") == []
def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None:
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
assert verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") is None
assert verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: None), _spec(), function="f") == []
def test_the_verdict_is_deterministic() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}]
cpg = _cpg(rows)
assert verdict(cpg, _spec(), function="login") == verdict(cpg, _spec(), function="login")
assert verdicts(cpg, _spec(), function="login")[0] == verdicts(cpg, _spec(), function="login")[0]
# --- framework coverage and the weak-algorithm arm --------------------------------------------------------
@@ -113,24 +113,24 @@ def test_the_settings_table_covers_the_frameworks_in_the_corpus() -> None:
def test_a_weak_algorithm_is_entailed_by_its_literal() -> None:
"""CWE-327: the danger is the algorithm NAMED, not a permissive flag."""
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung
from openultrasast.model.specs import config_specs
spec = config_specs(language="python")["config_secrets"]
rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}]
answer = verdict(_cpg(rows), spec, function="digest")
answer = verdicts(_cpg(rows), spec, function="digest")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
assert "md5" in answer.witness.lower()
def test_a_strong_algorithm_yields_no_finding() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
from openultrasast.model.specs import config_specs
spec = config_specs(language="python")["config_secrets"]
rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}]
assert verdict(_cpg(rows), spec, function="digest") is None
assert verdicts(_cpg(rows), spec, function="digest") == []
def _javascript_spec(): # type: ignore[no-untyped-def]
@@ -142,7 +142,7 @@ def _javascript_spec(): # type: ignore[no-untyped-def]
def test_disabling_template_escaping_is_entailed() -> None:
"""The setting NodeGoat's cross-site scripting turns off, claimed where the evidence is: the value."""
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung
rows = [
@@ -154,7 +154,7 @@ def test_disabling_template_escaping_is_entailed() -> None:
"args": ["{ autoescape: false }"],
}
]
answer = verdict(_cpg(rows), _javascript_spec(), function="<module>")
answer = verdicts(_cpg(rows), _javascript_spec(), function="<module>")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
assert "swig.setDefaults" in answer.witness and "false" in answer.witness
@@ -167,7 +167,7 @@ def test_enabling_template_escaping_is_not_a_finding() -> None:
reported on exactly the code that repairs it -- the commented-out `autoescape: true` two lines below
NodeGoat's defect.
"""
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
rows = [
{
@@ -178,12 +178,12 @@ def test_enabling_template_escaping_is_not_a_finding() -> None:
"args": ["{ autoescape: true }"],
}
]
assert verdict(_cpg(rows), _javascript_spec(), function="<module>") is None
assert verdicts(_cpg(rows), _javascript_spec(), function="<module>") == []
def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
"""The control for the other direction: scoping must not quiet the settings it was not about."""
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
from openultrasast.model.ladder import Rung
rows = [
@@ -195,7 +195,7 @@ def test_a_setting_whose_polarity_is_unchanged_still_reports_true() -> None:
"args": ["{ origin: true }"],
}
]
answer = verdict(_cpg(rows), _javascript_spec(), function="createApp")
answer = verdicts(_cpg(rows), _javascript_spec(), function="createApp")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
@@ -206,13 +206,13 @@ def test_the_first_setting_in_a_file_is_the_one_reported() -> None:
spelling of its line numbers. NodeGoat has a permissive session at line 78 and template escaping disabled
at line 135, and which one surfaced flipped on that comparison.
"""
from openultrasast.model.config_value import verdict
from openultrasast.model.config_value import verdicts
rows = [
{"setting": "swig.setDefaults({ autoescape: false })", "line": "135", "method": "m", "literalArgs": ["false"], "args": []},
{"setting": "cors({ origin: true })", "line": "78", "method": "m", "literalArgs": ["true"], "args": []},
]
answer = verdict(_cpg(rows), _javascript_spec(), function="m")
answer = verdicts(_cpg(rows), _javascript_spec(), function="m")[0]
assert answer is not None and "line 78" in answer.witness, answer.witness
tests/test_model_report.py
-90
View File
@@ -1,90 +0,0 @@
"""model-grounded-detection task 5.2: reporting that cannot flatter itself (Req 9.3, 11.1).
Three disciplines survive from the deleted publish/scoring layer, and each exists because its absence caused a
real error in this project's history:
per-slice rows a headline over mixed slices hid that agent-written code scored 44% while the
hand-written development slice scored 83%
the overfitting gap published beside the headline, so the distance between the development slice and the
real-world one is never something a reader has to compute
the rung on every a finding's strength is what established it; a report that omits the rung invites the
finding reader to treat a suspicion as a result
"""
from __future__ import annotations
import pytest
def _slice(name, pairs, covered): # type: ignore[no-untyped-def]
from openultrasast.model.report import SliceRow
return SliceRow(slice=name, pairs=pairs, covered=covered)
def test_the_report_carries_a_row_per_slice() -> None:
from openultrasast.model.report import render
text = render([_slice("vibe-py", 30, 25), _slice("vfc-js", 17, 4)])
assert "vibe-py" in text and "vfc-js" in text
assert "30" in text and "17" in text
def test_a_bare_aggregate_is_refused() -> None:
"""Req 11.1: no cross-slice number without the rows it was computed from."""
from openultrasast.model.report import render
with pytest.raises(ValueError, match="per-slice"):
render([])
def test_the_overfitting_gap_is_published_beside_the_headline() -> None:
"""The development slice minus the deciding real-world slice, stated rather than left to be computed."""
from openultrasast.model.report import overfitting_gap, render
rows = [_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)]
gap = overfitting_gap(rows, development="vibe-py", deciding="vfc-js")
assert gap == pytest.approx(0.80 - 0.20)
assert "overfitting gap" in render(rows, development="vibe-py", deciding="vfc-js").lower()
assert "+60" in render(rows, development="vibe-py", deciding="vfc-js")
def test_the_gap_is_absent_rather_than_zero_when_a_slice_is_missing() -> None:
"""A missing slice must not read as "no overfitting" — that is the flattering failure, not a neutral one."""
from openultrasast.model.report import overfitting_gap
assert overfitting_gap([_slice("vibe-py", 30, 24)], development="vibe-py", deciding="vfc-js") is None
def test_every_finding_shows_its_rung() -> None:
from openultrasast.findings import StaticFinding
from openultrasast.model.ladder import Rung
from openultrasast.model.report import render_findings
finding = StaticFinding(
finding_id="r:app.py:1",
path="app.py",
title="t",
severity="high",
confidence="medium",
evidence_level="static_corroboration",
rationale="why",
line=1,
function_name="run",
reachability_status="unknown",
reachability_evidence=[],
reachability_conditions=[],
tags=[],
ranking_priority=1.0,
rung=Rung.ENTAILED,
)
text = render_findings([finding])
assert "model_entailed" in text and "app.py" in text
def test_the_deciding_slice_is_named_in_the_report() -> None:
"""Req 9.3: the real-world slice decides, and the report says which one that is."""
from openultrasast.model.report import render
text = render([_slice("vibe-py", 30, 24), _slice("vfc-js", 20, 4)], development="vibe-py", deciding="vfc-js")
assert "vfc-js" in text and "decid" in text.lower()
tests/test_model_taint.py
+43 -45
View File
@@ -30,12 +30,12 @@ def _spec(): # type: ignore[no-untyped-def]
def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3}
]
answer = verdict(_cpg(rows), _spec(), function="run")
answer = verdicts(_cpg(rows), _spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
assert answer.family == "injection"
assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found"
@@ -44,30 +44,30 @@ def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None:
"""The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's."""
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}]
answer = verdict(_cpg(rows), _spec(), function="run")
answer = verdicts(_cpg(rows), _spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.CORROBORATED
def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None:
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
assert verdict(_cpg([]), _spec(), function="run") is None
assert verdicts(_cpg([]), _spec(), function="run") == []
def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None:
"""`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3."""
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None)
assert verdict(broken, _spec(), function="run") is None
assert verdicts(broken, _spec(), function="run") == []
def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -79,20 +79,20 @@ def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
"length": 2,
}
]
assert verdict(_cpg(rows), _spec(), function="run") is None, "a flow in another function does not judge this one"
assert verdicts(_cpg(rows), _spec(), function="run") == [], "a flow in another function does not judge this one"
def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None:
"""Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average."""
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9},
{"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2},
]
first = verdict(_cpg(rows), _spec(), function="run")
second = verdict(_cpg(rows), _spec(), function="run")
first = verdicts(_cpg(rows), _spec(), function="run")[0]
second = verdicts(_cpg(rows), _spec(), function="run")[0]
assert first == second
assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness
@@ -100,7 +100,7 @@ def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic()
def test_the_spec_drives_the_query_parameters() -> None:
"""The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam."""
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
seen: dict[str, object] = {}
@@ -109,7 +109,7 @@ def test_the_spec_drives_the_query_parameters() -> None:
seen.update(params)
return []
verdict(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run")
verdicts(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run")
assert seen["query"] == "taint"
assert seen["sources"] == ("request.args",)
assert seen["sinks"] == ("os.system", "execute")
@@ -143,7 +143,7 @@ def _shape_spec(): # type: ignore[no-untyped-def]
def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_reaches_it() -> None:
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -157,12 +157,12 @@ def test_a_parameterised_sink_call_is_not_a_vulnerability_even_though_the_flow_r
"sinkArg0Literal": True,
}
]
assert verdict(_cpg(rows), _shape_spec(), function="run") is None, "a bound query is the fix, not the bug"
assert verdicts(_cpg(rows), _shape_spec(), function="run") == [], "a bound query is the fix, not the bug"
def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -176,14 +176,14 @@ def test_an_interpolated_sink_call_of_the_same_name_is_entailed() -> None:
"sinkArg0Literal": False,
}
]
answer = verdict(_cpg(rows), _shape_spec(), function="run")
answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> None:
"""`os.system(x)` with one argument is not "safe because it has one argument" -- the rule is per sink."""
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -197,17 +197,17 @@ def test_the_shape_test_only_applies_to_sinks_declared_safe_in_that_shape() -> N
"sinkArg0Literal": False,
}
]
answer = verdict(_cpg(rows), _spec(), function="run") # os.system is not in safe_shape_sinks
answer = verdicts(_cpg(rows), _spec(), function="run")[0] # os.system is not in safe_shape_sinks
assert answer is not None and answer.rung is Rung.ENTAILED
def test_a_row_without_shape_fields_is_judged_on_the_flow_alone() -> None:
"""Older rows, or an engine that could not report arity, must not be silently treated as safe."""
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [{"sink": "db.execute(q)", "sinkLine": "5", "sinkMethod": "run", "source": "request.args['n']", "sanitized": False, "length": 3}]
answer = verdict(_cpg(rows), _shape_spec(), function="run")
answer = verdicts(_cpg(rows), _shape_spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
@@ -219,7 +219,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
So the flag must exist and must default to off.
"""
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
seen: dict[str, object] = {}
@@ -227,9 +227,9 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
seen.update(params)
return []
verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run")
verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run")
assert seen["parameterSources"] == "false", "treating every parameter as untrusted is not the default"
verdict(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True)
verdicts(CpgResult(cpg_path=Path("c.bin"), run=run), _spec(), function="run", parameter_sources=True)
assert seen["parameterSources"] == "true"
@@ -250,7 +250,7 @@ def test_parameter_sources_are_opt_in_and_reach_the_query() -> None:
def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None:
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -263,13 +263,13 @@ def test_a_sink_in_a_nested_closure_is_in_scope_when_the_query_says_so() -> None
"inLabeledScope": True,
}
]
answer = verdict(_cpg(rows), _spec(), function="requestListener")
answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
assert "<lambda>0" not in answer.witness, "the witness names the flow, not the synthetic closure name"
def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -282,17 +282,17 @@ def test_a_sink_in_an_unrelated_function_stays_out_of_scope() -> None:
"inLabeledScope": False,
}
]
assert verdict(_cpg(rows), _spec(), function="requestListener") is None
assert verdicts(_cpg(rows), _spec(), function="requestListener") == []
def test_rows_without_the_scope_flag_fall_back_to_the_method_name() -> None:
"""A row from an engine that cannot report scope must not silently become in-scope."""
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{"sink": "os.system(x)", "sinkLine": "9", "sinkMethod": "<lambda>0", "source": "request.args['x']", "sanitized": False, "length": 2}
]
assert verdict(_cpg(rows), _spec(), function="run") is None
assert verdicts(_cpg(rows), _spec(), function="run") == []
def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter() -> None:
@@ -305,7 +305,7 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter(
breaks ties, so the order stays total and the verdict deterministic.
"""
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -336,10 +336,10 @@ def test_a_flow_from_a_real_source_outranks_a_shorter_one_from_a_bare_parameter(
"inLabeledScope": True,
},
]
answer = verdict(_cpg(rows), _spec(), function="requestListener")
answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
assert "request.args['f']" in answer.witness, f"witness named no real source: {answer.witness}"
assert verdict(_cpg(rows), _spec(), function="requestListener") == answer # still deterministic
assert verdicts(_cpg(rows), _spec(), function="requestListener")[0] == answer # still deterministic
def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitized() -> None:
@@ -351,7 +351,7 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize
modelled source reaches the sink, those flows are the evidence and the bare-parameter ones are ignored.
"""
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -373,14 +373,14 @@ def test_a_modelled_source_flow_decides_even_when_a_parameter_flow_is_unsanitize
"inLabeledScope": True,
},
]
answer = verdict(_cpg(rows), _spec(), function="requestListener")
answer = verdicts(_cpg(rows), _spec(), function="requestListener")[0]
assert answer is not None and answer.rung is Rung.CORROBORATED, "a sanitized real flow is not an entailment"
assert "request.args['f']" in answer.witness
def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink() -> None:
from openultrasast.model.ladder import Rung
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
rows = [
{
@@ -393,7 +393,7 @@ def test_parameter_flows_still_decide_when_no_modelled_source_reaches_the_sink()
"inLabeledScope": True,
}
]
answer = verdict(_cpg(rows), _spec(), function="run")
answer = verdicts(_cpg(rows), _spec(), function="run")[0]
assert answer is not None and answer.rung is Rung.ENTAILED
@@ -443,13 +443,13 @@ def test_a_bound_that_governs_the_sink_lowers_the_rung() -> None:
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.ladder import Rung
from openultrasast.model.specs import taint_specs
from openultrasast.model.taint import verdict
from openultrasast.model.taint import verdicts
spec = taint_specs(language="c")["memory"]
assert "strcpy" in spec.bounded_sinks, "a CWE-121 sink is dischargeable by a bound"
governed = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main")
ungoverned = verdict(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main")
governed = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(True)), spec, function="main")[0]
ungoverned = verdicts(CpgResult(cpg_path=Path("c.bin"), run=lambda q, p: _bounded_rows(False)), spec, function="main")[0]
assert governed is not None and governed.rung is Rung.CORROBORATED
assert "governed by" in governed.witness, "the witness names the bound it found"
@@ -471,7 +471,7 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None:
verdict, and `system` won it on flow length, so the SQL injection went unreported by construction."""
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.specs import taint_specs
from openultrasast.model.taint import verdict, verdicts
from openultrasast.model.taint import verdicts
def row(sink, line, length): # type: ignore[no-untyped-def]
return {
@@ -497,5 +497,3 @@ def test_a_region_reports_every_sink_site_not_just_the_strongest() -> None:
assert len(every) == 2, "two sink sites are two findings"
assert {v.location for v in every} == {"app.php:6:handler", "app.php:13:handler"}
# The pair path is unchanged: the strongest is still what `verdict` returns.
assert every[0].location == verdict(cpg, spec, function="", file="app.php").location
tests/test_model_taxonomy.py
-16
View File
@@ -62,22 +62,6 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None
assert taxonomy.family_of_cwe("CWE-99999") is None
def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None:
from openultrasast.model.taxonomy import load_families
taxonomy = load_families()
assert taxonomy.related("injection", "injection") == "same"
assert taxonomy.related("injection", "access_control") == "lateral"
assert taxonomy.related("injection", "not_a_family") == "fabricated"
assert taxonomy.related("unknown", "injection") == "lateral" # abstention earns no partial credit
assert all(family.parent is None for family in taxonomy.families) # the shipped taxonomy is flat
(tmp_path / "families.toml").write_text(
GOOD + '\n[[family]]\nid = "path"\nparent = "injection"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "canary"\n'
)
scoped = load_families(tmp_path / "families.toml", require_all=False)
assert scoped.related("path", "injection") == "parent_child" and scoped.related("injection", "path") == "parent_child"
@pytest.mark.parametrize(
("text", "needle"),
[
tests/test_plane_delivery.py
-10
View File
@@ -129,16 +129,6 @@ def test_state_is_mirrored_and_seeds_a_rerun_elsewhere(store: HttpObjectStore, t
assert second.load_state() is None
def test_put_input_stores_a_file_no_task_produces(store: HttpObjectStore, tmp_path: Path) -> None:
delivery = Delivery(store, "r", tmp_path / "r")
source = tmp_path / "candidates.json"
source.write_bytes(b"[1, 2, 3]")
url = delivery.put_input("candidates.json", source)
assert store.objects["runs/r/inputs/candidates.json"] == b"[1, 2, 3]"
with urllib.request.urlopen(url, timeout=10) as response: # noqa: S310
assert response.read() == b"[1, 2, 3]"
def test_declared_artifacts_are_outputs_plus_what_consumers_name(tmp_path: Path) -> None:
"""A consumer may declare a producer's summary.json (every task writes one) without the producer listing it."""
from openultrasast.plane.manifests import load_manifests
tests/test_plane_egress.py
+3 -58
View File
@@ -3,8 +3,7 @@
Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the memory store's
host (where the runner PUTs its tar and GETs its inputs), the bound Workspaces' Git hosts and the bound Model's API
host, all as one TLS passthrough rule on 443 (plane-on-kubernetes 2.3), and nothing a gateway would reject or that
would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule (the receiver path, kept for
group 2 only, is the one exception).
would intercept TLS: no IP address, no wildcard, no ``https`` rule, no ``http`` rule.
"""
from __future__ import annotations
@@ -12,12 +11,11 @@ from __future__ import annotations
import ipaddress
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from openultrasast.plane import egress, runner
from openultrasast.plane import egress
from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest
MANIFESTS = """
@@ -63,7 +61,7 @@ def hostnames(policy: dict) -> list[str]:
def assert_tight(policy: dict) -> None:
for rule in policy["rules"]:
assert set(rule) <= {"http", "tlsPassthrough"}, f"only http and passthrough rules, got {rule}"
assert set(rule) == {"tlsPassthrough"}, f"only TLS passthrough rules, got {rule}"
body = next(iter(rule.values()))
assert "all" not in body["ports"], "never every port"
for host in hostnames(policy):
@@ -82,12 +80,6 @@ def test_model_free_task_gets_only_the_store(manifests) -> None:
assert egress.policy_for(manifests.tasks["facts"], [], None, None) == {"rules": []}, "no store, no Git, no Model: nothing"
def test_the_receiver_path_adds_the_one_http_rule_until_it_is_removed(manifests) -> None:
policy = egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host=egress.RECEIVER_HOST)
assert policy == {"rules": [{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}}]}
assert egress.policy_for(manifests.tasks["facts"], [], None, None, receiver_host="127.0.0.1") == {"rules": []}
def test_model_bound_task_with_git_workspaces(manifests) -> None:
task = manifests.tasks["verify"]
policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], STORE)
@@ -191,50 +183,3 @@ def test_apply_waits_for_the_actor_then_gives_up(tmp_path: Path, kubectl_ate: Pa
timer.start()
cli.apply("late", "default", POLICY, timeout=10)
timer.join()
def test_receiver_address_modes(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
assert egress.receiver_address(4242, None) == ("http://127.0.0.1:4242/", None)
monkeypatch.delenv("OUSAST_ARTIFACT_HOST")
monkeypatch.setenv("OUSAST_ARTIFACT_DIAL", "172.19.0.1:80")
assert egress.receiver_address(18090, "kind-test") == (f"http://{egress.RECEIVER_HOST}/", "172.19.0.1:80")
monkeypatch.delenv("OUSAST_ARTIFACT_DIAL")
with pytest.raises(RuntimeError, match="kube context"):
egress.receiver_address(18090, None) # no context to look the Service up in: never a built one
# --- the runner dials the gateway address with the receiver's name as Host ----------------------------------------
class _Seen(ThreadingHTTPServer):
daemon_threads = True
seen: list[tuple[str, str | None, bytes]]
class _Handler(BaseHTTPRequestHandler):
server: _Seen
def do_POST(self) -> None: # noqa: N802
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
self.server.seen.append((self.path, self.headers.get("Host"), body))
self.send_response(200)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
def test_delivery_dials_the_override_with_the_url_host() -> None:
server = _Seen(("127.0.0.1", 0), _Handler)
server.seen = []
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
dial = f"127.0.0.1:{server.server_address[1]}"
url = f"http://{egress.RECEIVER_HOST}/in"
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, dial=dial) == 200
assert server.seen == [("/in", egress.RECEIVER_HOST, b"tar")]
finally:
server.shutdown()
server.server_close()
tests/test_plane_features.py
+1 -34
View File
@@ -9,10 +9,9 @@ from pathlib import Path
import pytest
from openultrasast.learn.schema import FeatureRecordError, feature_set_digest
from openultrasast.plane.memory import FileStore
from openultrasast.plane.tasks import features
from openultrasast.plane.tasks.remember import Context, remember_run, rows_for
from openultrasast.plane.tasks.remember import remember_run
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
CASE = "lmdeploy-media-url-ssrf"
@@ -25,30 +24,6 @@ def run_dir(tmp_path: Path) -> Path:
return target
def _checkout(root: Path) -> Path:
media = root / "lmdeploy" / "vl" / "media"
media.mkdir(parents=True)
(media / "connection.py").write_text("def _is_safe_url(u):\n return True\n\n\ndef _load_http_url(u):\n x = 1\n return x\n")
return root
def test_features_of_the_recorded_case(run_dir: Path, tmp_path: Path) -> None:
[rec] = features.recorded_case(run_dir, CASE, workspace=_checkout(tmp_path / "checkout"))
assert (rec["candidate"], rec["family"], rec["profile"], rec["unit"]) == (
"lmdeploy/vl/media/connection.py::_load_http_url", "untrusted_destination", "plane", "pin",
) # fmt: skip
x, states = rec["x"], {k: v["state"] for k, v in rec["instruments"].items()}
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, True, None, 2)
assert x["agree.final"] == "agreed" and x["facts.callers"] == 0 and x["facts.is_global"] is False
assert x["facts.function_lines"] == 3 and x["lang.python"] is True and x["verify.site_offset"] == 0
assert states["verify"] == states["agree"] == states["facts"] == states["source"] == "ran"
assert states["quick"] == states["engine"] == "none", "no alerts task in this Run: no coverage, not zero hits"
assert rec["instruments"]["verify"]["version"] == "deepseek-flash" and rec["feature_set_digest"] == feature_set_digest()
assert "site_match" not in json.dumps(rec), "the fixture's site_match: true was dropped on read"
[without] = features.recorded_case(run_dir, CASE)
assert without["instruments"]["source"]["state"] == "none" and without["x"]["facts.function_lines"] is None
def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None:
out = tmp_path / "out"
env = {
@@ -71,13 +46,5 @@ def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_pat
assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed"
def test_remember_refuses_a_record_outside_the_allow_list(run_dir: Path) -> None:
[rec] = features.recorded_case(run_dir, CASE)
rec["x"]["site_match"] = True
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
with pytest.raises(FeatureRecordError, match="site_match"):
rows_for(ctx, facts=None, passes={}, models={}, agreed=None, features=[rec])
def test_the_task_fails_loudly_without_an_output_dir() -> None:
assert features.main({}) == 2
tests/test_plane_increment.py
+1 -1
View File
@@ -77,7 +77,7 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls
env = {e.name: e.value for e in task.env}
assert env["OUSAST_PASS"] == entry.name[-1]
docs = reconciler.render_task(run, entry, manifests, "http://127.0.0.1:1/")
docs = reconciler.render_task(run, entry, manifests)
for doc in docs:
assert not non_ax_fields(doc["spec"], AX_FIELDS[doc["kind"]], doc["kind"])
assert set(doc["metadata"]) <= {"name", "atespace"}
tests/test_plane_reconciler.py
+7 -94
View File
@@ -5,7 +5,7 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates
fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
-- and on the scripted poll it PUTs a scripted tar to the presigned ``delivery.put`` URL the start request carried
(plane-on-kubernetes 2.4; or POSTs it to ``OUSAST_ARTIFACT_URL`` in receiver mode), or turns ``Failed`` with a
(plane-on-kubernetes 2.4), or turns ``Failed`` with a
condition message, or vanishes. Every outcome comes from a scripted delivery, and a delivery happens only after
the reconciler's start request reached the fake router (Req 4.5). The store is ``plane_fake_store.HttpObjectStore``.
"""
@@ -21,8 +21,6 @@ import sys
import tarfile
import threading
import time
import urllib.error
import urllib.request
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
@@ -152,11 +150,9 @@ elif argv[:2] == ["get", "task"]:
delivery = body.get("delivery") or {{}}
if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
files = script.get("files", {{}})
base = env.get("OUSAST_ARTIFACT_URL", "").rstrip("/") + "/inputs/"
for name, ref in json.loads(env.get("OUSAST_INPUTS", "{{}}")).items():
presigned = (delivery.get("inputs") or {{}}).get(name)
fetch = urllib.request.Request(presigned) if presigned else urllib.request.Request(
base + ref, headers={{"X-Ousast-Run": env["OUSAST_RUN"], "X-Ousast-Task": task}})
fetch = urllib.request.Request(presigned)
try:
with urllib.request.urlopen(fetch) as resp:
dest = HERE / "fetched" / task / ref
@@ -175,13 +171,8 @@ elif argv[:2] == ["get", "task"]:
info = tarfile.TarInfo(path)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
headers = {{"Content-Type": "application/x-tar", "X-Ousast-Run": env["OUSAST_RUN"]}}
headers["X-Ousast-Task"] = script.get("as_task", task)
if delivery.get("put"): # the store path: PUT to the presigned URL, no receiver headers
tar_headers = {{"Content-Type": "application/x-tar"}}
req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers)
else:
req = urllib.request.Request(env["OUSAST_ARTIFACT_URL"], data=buf.getvalue(), method="POST", headers=headers)
tar_headers = {{"Content-Type": "application/x-tar"}}
req = urllib.request.Request(delivery["put"], data=buf.getvalue(), method="PUT", headers=tar_headers)
try:
with urllib.request.urlopen(req) as resp:
(HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n")
@@ -420,7 +411,6 @@ def test_chain_runs_in_order_with_inputs_env_and_model(fake: Fake, tmp_path: Pat
assert env["OUSAST_BUDGET_USD"] == "2.5" and env["OUSAST_BUDGET_CALLS"] == "40"
assert env["OUSAST_INPUT_FACTS"] == "/workspace/.ousast-in/verify/repo-facts/facts.json"
assert json.loads(env["OUSAST_INPUTS"]) == {"FACTS": "repo-facts/facts.json"}
assert "OUSAST_ARTIFACT_URL" not in env, "the store path renders no receiver address into the Task"
assert env["OUSAST_MODEL"] == "deepseek-flash"
assert json.loads(env["OUSAST_MODEL_PARAMS"]) == {"cache_hit_per_m": 0.07, "input_per_m": 0.27, "output_per_m": 1.1}
kinds = [(d["kind"], d["metadata"]["name"]) for d in verify["docs"]]
@@ -661,33 +651,6 @@ def tar_of(files: dict[str, bytes]) -> bytes:
return buf.getvalue()
def test_receiver_rejects_traversal_unknown_tasks_and_wrong_types(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") # the direct test address: no Service lookup, no context needed
receiver = reconciler.Receiver("r", tmp_path / "r", 0)
receiver.running.add("t")
headers = {"X-Ousast-Run": "r", "X-Ousast-Task": "t", "Content-Type": "application/x-tar"}
code, text = receiver.accept(headers, tar_of({"../escape.json": b"{}"}))
assert code == 400 and "escape" in text and not (tmp_path / "escape.json").exists()
assert receiver.accept({**headers, "X-Ousast-Task": "other"}, tar_of({"summary.json": b"{}"}))[0] == 403
assert receiver.accept({**headers, "X-Ousast-Run": "someone"}, tar_of({"summary.json": b"{}"}))[0] == 403
assert receiver.accept({**headers, "Content-Type": "application/json"}, b"{}")[0] == 415
assert receiver.accept(headers, b"not a tar")[0] == 400
assert receiver.accept(headers, tar_of({"summary.json": b"{}", "sub/x.json": b"1"}))[0] == 200
assert (tmp_path / "r" / "t" / "sub" / "x.json").read_text() == "1" and "t" in receiver.delivered
receiver.server_close()
def test_delivery_for_a_task_that_is_not_running_is_rejected(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_DELIVERY", "receiver") # the receiver path, kept for group 2 (task 2.6 removes this test)
monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "1") # the rejected delivery is no delivery: the task times out
script = json.loads(json.dumps(SCRIPT))
script["repo-facts"]["as_task"] = "agree" # the runner claims to be a task that has not started
fake.script(script)
assert reconciler.run(write_run(tmp_path, "spoof"), ax=str(fake.ax)) == "failed"
assert [n for _, n in fake.events("rejected")] == ["repo-facts"]
assert not (reconciler.run_dir("spoof") / "agree" / "summary.json").exists()
def test_lock_refuses_a_second_runner_and_ignores_a_dead_one(fake: Fake, tmp_path: Path) -> None:
manifest = write_run(tmp_path, "locked")
base = reconciler.run_dir("locked")
@@ -824,7 +787,7 @@ def test_rendered_documents_carry_only_ax_fields(tmp_path: Path) -> None:
(tmp_path / "repo-facts").mkdir()
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
run, manifests = reconciler.load_run(write_run(tmp_path, "fields"))
docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests, "http://h:1/")]
docs = [d for entry in run.tasks for d in reconciler.render_task(run, entry, manifests)]
assert {d["kind"] for d in docs} == {"Task", "Workspace", "Model"}
for rendered in docs:
where = f"{rendered['kind']}/{rendered['metadata']['name']}"
@@ -841,7 +804,7 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None:
(tmp_path / "repo-facts" / "facts.json").write_text("{}", encoding="utf-8")
run, manifests = reconciler.load_run(write_run(tmp_path, "order"))
for entry in run.tasks:
docs = reconciler.render_task(run, entry, manifests, "http://h:1/")
docs = reconciler.render_task(run, entry, manifests)
assert [d["kind"] for d in docs].count("Task") == 1
assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs]
@@ -943,44 +906,6 @@ def test_a_refused_egress_policy_fails_the_task_before_resume(fake: Fake, tmp_pa
SUBSTRATE_ENV_LIMIT = 32768 # "actor_template.containers[0].env[11].value: Too long" on the live cluster, 2026-09-29
def get_input(receiver: reconciler.Receiver, path: str, run: str = "r", task: str = "t") -> tuple[int, bytes]:
url = f"http://127.0.0.1:{receiver.server_address[1]}{path}"
request = urllib.request.Request(url, headers={"X-Ousast-Run": run, "X-Ousast-Task": task})
try:
with urllib.request.urlopen(request, timeout=10) as response:
return response.status, response.read()
except urllib.error.HTTPError as exc:
return exc.code, exc.read()
def test_receiver_serves_a_running_task_its_declared_inputs_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
base = tmp_path / "r"
(base / "p").mkdir(parents=True)
big = b"x" * (3 << 20)
(base / "p" / "a.json").write_bytes(big)
(base / "p" / "b.json").write_bytes(b"not declared")
(tmp_path / "secret.txt").write_text("outside the run")
receiver = reconciler.Receiver("r", base, 0)
threading.Thread(target=receiver.serve_forever, daemon=True).start()
try:
receiver.running.update({"t", "idle"})
receiver.inputs.update({"t": frozenset({"p/a.json", "p/missing.json", "../secret.txt"}), "done": frozenset({"p/a.json"})})
assert get_input(receiver, "/inputs/p/a.json") == (200, big), "streamed whole"
assert get_input(receiver, "/inputs/p/a.json", run="other")[0] == 403, "wrong run"
assert get_input(receiver, "/inputs/p/a.json", task="done")[0] == 403, "a task that is not running"
assert get_input(receiver, "/inputs/p/a.json", task="idle")[0] == 403, "a running task without that input"
assert get_input(receiver, "/inputs/p/b.json")[0] == 403, "not among the task's inputs"
assert get_input(receiver, "/inputs/p/missing.json")[0] == 404, "declared but not delivered"
for escape in ("/inputs/../secret.txt", "/inputs/%2e%2e/secret.txt", "/inputs/p/../../secret.txt", "/inputs//etc/passwd"):
code, body = get_input(receiver, escape)
assert code in (403, 404) and b"outside" not in body, escape
assert get_input(receiver, "/p/a.json")[0] == 404, "only /inputs/ is served"
finally:
receiver.shutdown()
receiver.server_close()
def test_a_missing_producer_artifact_fails_the_consumer_instead_of_running_it(fake: Fake, tmp_path: Path) -> None:
script = json.loads(json.dumps(SCRIPT))
del script["repo-facts"]["files"]["facts.json"] # repo-facts reports done but never delivered facts.json
@@ -996,7 +921,7 @@ def test_a_megabyte_producer_artifact_keeps_every_rendered_value_under_substrate
(tmp_path / "repo-facts" / "facts.json").write_text(json.dumps({"blob": "f" * (1 << 20)}), encoding="utf-8")
run, manifests = reconciler.load_run(write_run(tmp_path, "big"))
for entry in run.tasks:
docs = reconciler.render_task(run, entry, manifests, "http://h:1/", "10.0.0.1:80")
docs = reconciler.render_task(run, entry, manifests)
task, workspaces = docs[-1], [d for d in docs if d["kind"] == "Workspace"]
assert all(len(e["value"]) < SUBSTRATE_ENV_LIMIT for e in task["spec"]["env"]), entry.name
assert len(yaml.safe_dump(task)) < SUBSTRATE_ENV_LIMIT, f"{entry.name}: AX_TASK_YAML"
@@ -1080,15 +1005,3 @@ def test_a_tar_the_store_path_cannot_accept_fails_the_task(fake: Fake, tmp_path:
assert not (reconciler.run_dir("esc").parent / "escape.json").exists()
def test_receiver_mode_still_runs_through_the_host_receiver(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""OUSAST_DELIVERY=receiver (group 2 only): the old path, with its http rule and OUSAST_ARTIFACT_URL, until 2.6."""
monkeypatch.setenv("OUSAST_DELIVERY", "receiver")
assert reconciler.run(write_run(tmp_path, "rcv"), ax=str(fake.ax)) == "done"
assert fake.applied("rcv", "verify")["env"]["OUSAST_ARTIFACT_URL"].startswith("http://127.0.0.1:")
assert [s["body"].get("delivery") for s in fake.router.starts] == [None, None, None], "no presigned URLs in receiver mode"
assert not any(k.startswith("runs/rcv/") and k.endswith("output.tar") for k in fake.store.objects), "nothing PUT to the store"
assert "runs/rcv/state.json" in fake.store.objects, "the state is still mirrored"
written = json.loads((tmp_path / "policies" / "default_rcv-repo-facts.written.json").read_text())
assert written["rules"] == [{"tlsPassthrough": {"hostnames": ["example.invalid"], "ports": {"numbers": [443]}}}], (
"an IP receiver: no http rule"
)
tests/test_plane_remember.py
+1 -1
View File
@@ -39,7 +39,7 @@ def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]:
return rows
def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
def test_remember_turns_the_saved_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
assert len(facts) > 100, "the fixture was read"
store = FileStore(tmp_path / "memory")
tests/test_plane_runner.py
+29 -217
View File
@@ -19,13 +19,12 @@ import time
import urllib.error
import urllib.request
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from plane_fake_store import HttpObjectStore
from openultrasast.plane import egress, runner
from openultrasast.plane import runner
from openultrasast.plane.manifests import GitSource
from openultrasast.plane.router import Router, StartError
from openultrasast.plane.runner import (
@@ -194,96 +193,10 @@ def test_main_serves_readiness_on_a_free_port_and_the_task_sees_200(
# --- artifact delivery (Requirement 4.4) --------------------------------------------------------------------------------
class Receiver(ThreadingHTTPServer):
"""Records every POST (headers and body) and answers with ``status``."""
def __init__(self, status: int) -> None:
super().__init__(("127.0.0.1", 0), _ReceiverHandler)
self.status = status
self.posts: list[tuple[dict[str, str], bytes]] = []
self.thread = threading.Thread(target=self.serve_forever, daemon=True)
self.thread.start()
@property
def url(self) -> str:
return f"http://127.0.0.1:{self.server_address[1]}/artifacts"
def stop(self) -> None:
self.shutdown()
self.server_close()
class _ReceiverHandler(BaseHTTPRequestHandler):
server: Receiver
def do_POST(self) -> None: # noqa: N802
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
self.server.posts.append(({k: v for k, v in self.headers.items()}, body))
self.send_response(self.server.status)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
@pytest.fixture
def receiver() -> Iterator[Receiver]:
server = Receiver(200)
try:
yield server
finally:
server.stop()
def test_output_dir_is_delivered_as_a_tar_with_the_headers(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, OUSAST_RUN="validation-46")
assert code == 0
assert len(receiver.posts) == 1
headers, body = receiver.posts[0]
assert headers["Content-Type"] == "application/x-tar"
assert headers["X-Ousast-Task"] == "stub"
assert headers["X-Ousast-Run"] == "validation-46"
receiver.posts.clear()
again = {"OUSAST_ARTIFACT_URL": receiver.url, "OUSAST_TASK": "facts", "OUSAST_STATE_DIR": str(tmp_path / "again-state")}
code, _ = run_main(monkeypatch, tmp_path / "again", files_workspace_yaml(), **again)
assert code == 0 and receiver.posts[0][0]["X-Ousast-Task"] == "facts", "the Run's task name, not the ax Task's"
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as archive:
names = sorted(archive.getnames())
summary = archive.extractfile("summary.json")
assert summary is not None
assert json.loads(summary.read())["status"] == "done"
assert names == ["facts.json", "summary.json"]
def test_run_header_is_absent_without_ousast_run(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url)
assert code == 0
assert "X-Ousast-Run" not in receiver.posts[0][0]
def test_delivery_failure_is_reported_as_failed_with_exit_2(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
broken = Receiver(500)
try:
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url)
finally:
broken.stop()
assert code == EXIT_BY_STATUS["failed"] == 2
assert len(broken.posts) == 3, "three attempts"
summary = json.loads((output / "summary.json").read_text())
assert summary["status"] == "failed"
assert summary["reason"].startswith("artifact delivery failed: HTTP 500")
assert summary["units_done"] == 1 and summary["units_total"] == 1, "the task's other fields are kept"
def test_deliver_retries_on_connection_errors_then_raises() -> None:
closed = Receiver(200)
url = closed.url
closed.stop()
closed = HttpObjectStore()
url = closed.presign_put("output.tar")
closed.close()
with pytest.raises(DeliveryError, match="3 attempts"):
deliver(url, b"x", {"Content-Type": "application/x-tar"}, backoff=0)
@@ -366,9 +279,9 @@ def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name:
return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], tmp_path: Path) -> None:
port = free_port()
proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url)
proc = spawn_runner(tmp_path, port)
marker = tmp_path / "state" / "stub.done.json"
try:
wait_for(marker.exists)
@@ -377,8 +290,7 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env:
base = f"http://127.0.0.1:{port}"
assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200
assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200
assert len(receiver.posts) == 1
assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": True, "summary": None}
assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": False, "summary": None}
proc.send_signal(signal.SIGTERM)
assert proc.wait(timeout=10) == 0
finally:
@@ -386,9 +298,9 @@ def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env:
proc.wait()
def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], tmp_path: Path) -> None:
port = free_port()
proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="sleep", OUSAST_TERM_GRACE="1")
proc = spawn_runner(tmp_path, port, STUB_MODE="sleep", OUSAST_TERM_GRACE="1")
pids_file = tmp_path / "out" / "pids.json"
try:
wait_for(pids_file.exists)
@@ -403,45 +315,21 @@ def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: di
proc.kill()
proc.wait()
assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete"
assert receiver.posts == [], "nothing is delivered for an interrupted command"
def test_the_completion_marker_prevents_a_rerun_on_resume(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
counter = tmp_path / "runs.txt"
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
assert code == 0 and len(counter.read_text().splitlines()) == 1 and len(receiver.posts) == 1
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter))
assert code == 0 and len(counter.read_text().splitlines()) == 1
(tmp_path / "ws" / "candidates.json").write_text("changed by the task\n")
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), STUB_COUNTER=str(counter))
assert code == 0
assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work"
assert len(receiver.posts) == 1, "a delivered result is not delivered again"
assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once"
def test_a_failed_delivery_is_retried_alone_on_the_next_boot(
env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
counter = tmp_path / "runs.txt"
broken = Receiver(500)
try:
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url, STUB_COUNTER=str(counter))
finally:
broken.stop()
assert code == 2
marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
assert marker["delivered"] is False and marker["status"] == "failed" and marker["exit"] == 0
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
assert code == 0
assert len(counter.read_text().splitlines()) == 1, "only the delivery is retried"
with tarfile.open(fileobj=io.BytesIO(receiver.posts[0][1]), mode="r:") as archive:
summary = archive.extractfile("summary.json")
assert summary is not None and json.loads(summary.read())["status"] == "done", "the task's own summary"
marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
assert marker["delivered"] is True and marker["status"] == "done"
# --- git workspaces from the case cache and by shallow clone ----------------------------------------------------------------
@@ -590,14 +478,13 @@ def test_git_commits_annotation_round_trips_to_the_checkout(
"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata: {name: rt}\nspec: {tasks: [{name: stub, task: stub}]}\n"
)
run, manifests = reconciler.load_run(manifest)
*rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests, "unused")
*rest, task_doc = reconciler.render_task(run, run.tasks[0], manifests)
assert rest[0]["metadata"] == {"name": "case"}, "annotations are stripped before ax"
assert rest[0]["spec"]["git"] == [{"name": "code", "repo": repo.as_uri(), "branch": "main"}], "no commit for ax"
overrides = {"OUSAST_OUTPUT_DIR": str(output)}
task_doc["spec"]["env"] = [
{"name": e["name"], "value": overrides.get(e["name"], e["value"])}
for e in task_doc["spec"]["env"]
if e["name"] != "OUSAST_ARTIFACT_URL"
]
assert json.loads({e["name"]: e["value"] for e in task_doc["spec"]["env"]}["OUSAST_GIT_PINS"]) == {"case/code": pin}
monkeypatch.setenv("AX_TASK_YAML", yaml.safe_dump(task_doc))
@@ -669,11 +556,11 @@ def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict
def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child(
env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture
env: dict[str, str], tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
caplog.set_level(logging.DEBUG)
counter = tmp_path / "runs.txt"
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
thread, environ, codes = main_in_thread(tmp_path, STUB_COUNTER=str(counter))
port = environ["AX_RUNNER_BOUND_PORT"]
time.sleep(0.5)
assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request"
@@ -693,15 +580,14 @@ def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only
assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there"
assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files"
assert files_containing(tmp_path, SECRET) == [], "no file holds the credential"
assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar"
assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record"
assert any("start request: 202" in record.getMessage() for record in caplog.records)
def test_a_crash_that_prints_the_credential_is_redacted(
env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str]
env: dict[str, str], tmp_path: Path, capfd: pytest.CaptureFixture[str]
) -> None:
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak")
thread, environ, codes = main_in_thread(tmp_path, STUB_MODE="leak")
start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202
thread.join(timeout=30)
@@ -727,23 +613,25 @@ def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch:
assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists()
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path) -> None:
"""Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed
actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start."""
counter = tmp_path / "runs.txt"
same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"}
same = {"STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1", "AX_RUNNER_INSECURE_DELIVERY": "1"}
real_port, golden_port = free_port(), free_port()
real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same)
golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same)
try:
for port in (real_port, golden_port):
wait_for(lambda port=port: ready_on(port))
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET})
Router(f"http://127.0.0.1:{real_port}").start_task(
"default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}, {"put": object_store.presign_put("output.tar")}
)
marker = tmp_path / "state" / "stub.done.json"
wait_for(marker.exists)
time.sleep(1.0)
assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command"
assert golden.poll() is None and len(receiver.posts) == 1
assert golden.poll() is None
with pytest.raises(StartError, match="already started"):
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {})
again_port = free_port()
@@ -762,10 +650,10 @@ def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: R
assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"
def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str], tmp_path: Path) -> None:
"""Substrate's golden boot never gets a start, so it must touch no workspace (and no network): a clone at boot
failed on the live cluster while the actor was still being restored (2026-09-29)."""
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url)
thread, environ, codes = main_in_thread(tmp_path)
time.sleep(0.5)
written = [p for p in (tmp_path / "ws").rglob("*") if p.is_file()] if (tmp_path / "ws").exists() else []
assert written == [], "nothing is materialised before the start request"
@@ -778,82 +666,6 @@ def test_workspaces_are_prepared_only_after_the_start_request(env: dict[str, str
# --- inputs other tasks produced: fetched from the receiver after the start (Requirement 3.1, 4.4) -------------------
@pytest.fixture
def plane_receiver(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[egress.Receiver]:
"""The reconciler's own receiver on a free port, with a producer's artifact delivered and the task ``stub`` running."""
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
server = egress.Receiver("r", tmp_path / "run", 0)
(tmp_path / "run" / "repo-facts").mkdir(parents=True)
(tmp_path / "run" / "repo-facts" / "facts.json").write_bytes(b'{"a.py": ["f"]}' * 5000)
server.running.add("stub")
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield server
finally:
server.shutdown()
server.server_close()
def input_env(server: egress.Receiver, tmp_path: Path) -> dict[str, str]:
return {
"OUSAST_ARTIFACT_URL": server.url,
"OUSAST_RUN": "r",
"OUSAST_TASK": "stub",
"OUSAST_INPUTS": json.dumps({"FACTS": "repo-facts/facts.json"}),
"OUSAST_INPUT_FACTS": str(tmp_path / "ws" / "in" / "repo-facts" / "facts.json"),
}
def test_inputs_are_fetched_from_the_receiver_before_the_command(
env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
plane_receiver.inputs["stub"] = frozenset({"repo-facts/facts.json"})
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path))
assert code == 0
fetched = tmp_path / "ws" / "in" / "repo-facts" / "facts.json"
assert fetched.read_bytes() == (tmp_path / "run" / "repo-facts" / "facts.json").read_bytes()
assert "in/repo-facts/facts.json" in json.loads((output / "facts.json").read_text())["workspace_files"], "there before the command ran"
assert "stub" in plane_receiver.delivered
def test_an_input_that_cannot_be_fetched_delivers_a_failed_summary_and_never_runs_the_command(
env: dict[str, str], plane_receiver: egress.Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
plane_receiver.inputs["stub"] = frozenset({"repo-facts/other.json"}) # facts.json is not among its inputs: 403
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), **input_env(plane_receiver, tmp_path))
assert code == EXIT_BY_STATUS["failed"]
assert not (output / "facts.json").exists(), "the command never ran with an input missing"
delivered = json.loads((tmp_path / "run" / "stub" / "summary.json").read_text())
assert delivered["status"] == "failed" and "input FACTS (repo-facts/facts.json) not fetched: HTTP 403" in delivered["reason"]
assert not (tmp_path / "ws" / "in" / "repo-facts").exists() or not any((tmp_path / "ws" / "in" / "repo-facts").iterdir())
def test_fetch_dials_the_override_and_sends_the_url_host(tmp_path: Path) -> None:
seen: list[dict[str, str]] = []
class _Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
seen.append({"path": self.path, **dict(self.headers.items())})
self.send_response(200)
self.send_header("Content-Length", "2")
self.end_headers()
self.wfile.write(b"ok")
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
dial = f"127.0.0.1:{server.server_address[1]}"
size = runner.fetch("http://receiver.example/inputs/p/a.json", {"X-Ousast-Run": "r"}, tmp_path / "x" / "a.json", dial=dial)
finally:
server.shutdown()
server.server_close()
assert size == 2 and (tmp_path / "x" / "a.json").read_bytes() == b"ok"
assert seen[0]["path"] == "/inputs/p/a.json" and seen[0]["Host"] == "receiver.example" and seen[0]["X-Ousast-Run"] == "r"
# --- store delivery through the start body (plane-on-kubernetes 2.2, design section 3) ---------------------------
@@ -905,7 +717,7 @@ def test_output_is_put_to_the_presigned_url_and_inputs_are_fetched_from_theirs(
env: dict[str, str], object_store: HttpObjectStore, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""End to end against the S3-shaped fake: the start carries the URLs, the runner PUTs the tar and GETs the input;
no receiver, no OUSAST_ARTIFACT_URL, no URL in any log line."""
no URL in any log line."""
caplog.set_level(logging.DEBUG)
object_store._put("runs/run-1/repo-facts/facts.json", b'{"a.py": ["f"]}')
put_url = object_store.presign_put("runs/run-1/stub/output.tar")
@@ -964,9 +776,9 @@ def test_a_failed_put_is_retried_alone_on_the_next_start_with_a_fresh_url(
def test_deliver_puts_and_names_only_the_safe_url_on_failure(object_store: HttpObjectStore) -> None:
url = object_store.presign_put("k/output.tar")
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, method="PUT") == 200
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0) == 200
assert object_store.objects["k/output.tar"] == b"tar"
with pytest.raises(runner.DeliveryError) as caught:
runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0, method="PUT")
runner.deliver(url.replace("/fake-bucket/", "/other/"), b"x", {}, attempts=2, backoff=0)
assert "HTTP 404" in str(caught.value) and "X-Amz" not in str(caught.value) and "/other/k/output.tar" in str(caught.value)
assert runner.safe_url("https://h:1/p/q?X-Amz-Signature=s") == "https://h:1/p/q"
tests/test_push_inputs.py
+3 -3
View File
@@ -6,7 +6,7 @@ import subprocess
import pytest
from openultrasast.push_inputs import validate_manifest
from benchmarks.push.push_inputs import validate_manifest
def git(root, *args):
@@ -118,7 +118,7 @@ def test_authored_edit_requires_exact_original_and_provenance(manifest):
@pytest.mark.parametrize("field", ["snapshots", "cases"])
def test_empty_population_rejected(manifest, field):
from openultrasast.push_inputs import InputError
from benchmarks.push.push_inputs import InputError
data, _, run = manifest
data[field] = []
@@ -127,7 +127,7 @@ def test_empty_population_rejected(manifest, field):
def test_missing_snapshot_identity_rejected(manifest):
from openultrasast.push_inputs import InputError
from benchmarks.push.push_inputs import InputError
data, _, run = manifest
del data["snapshots"][0]["id"]
tests/test_rank.py
+1 -21
View File
@@ -5,14 +5,7 @@ import pytest
from openultrasast.preprocess import FileTarget
from openultrasast.provider.openrouter import OpenRouterError, parse_json_content
from openultrasast.rank import (
RankingError,
composite_priority,
parse_ranker_response,
rank_targets,
rank_targets_with_model,
write_rankings,
)
from openultrasast.rank import RankingError, composite_priority, parse_ranker_response, rank_targets, write_rankings
class FakeRankerClient:
@@ -72,19 +65,6 @@ def test_parse_ranker_response_rejects_malformed_scores() -> None:
parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model")
def test_rank_targets_with_model_chunks_requests() -> None:
targets = [
FileTarget(path="a.py", absolute_path="/repo/a.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
FileTarget(path="b.py", absolute_path="/repo/b.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
]
client = FakeRankerClient()
rankings = rank_targets_with_model(targets, client=client, model="openrouter/test-ranker", chunk_size=1)
assert client.calls == 2
assert {ranking.model_id for ranking in rankings} == {"openrouter/test-ranker"}
def test_parse_json_content_accepts_fenced_json() -> None:
assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []}
tests/test_search_coordinator.py
-27
View File
@@ -169,33 +169,6 @@ def test_claim_is_visible_in_worker_snapshot(tmp_path):
Coordinator(board(tmp_path), execute, budget=SearchBudget(reason_rounds=1)).run()
def test_batch_adapter_offline_checkpoint_and_step_dispatch(tmp_path, monkeypatch):
from dataclasses import asdict
from pathlib import Path
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1]))
from openultrasast.search.coordinator import BatchExecutor, SearchTask
calls = []
def lane(item, output, attempt):
import json
spec = json.loads(item.inputs["SPEC_URL"])
calls.append(spec)
assert item.extra_env["SEARCH_STEP"] == spec["step"]
return {"status": "ok", "cost_usd": 0, "intents": []}
adapter = BatchExecutor(
image="registry/task@sha256:" + "a" * 64, command=("python3", "/worker.py"), output=tmp_path, lanes={"ax": lane}
)
for index, step in enumerate(("reason", "explore", "verify")):
task = SearchTask(f"task-{index}", "search-1", step, "facts: []", {}, asdict(SearchBudget()))
assert adapter(task)["status"] == "ok"
assert adapter(task)["status"] == "ok"
assert len(calls) == 3
def test_invalid_identifiers_refused_before_paths(tmp_path):
from dataclasses import asdict
tests/test_verification.py
-11
View File
@@ -10,22 +10,11 @@ from openultrasast.verification import (
VerificationStatus,
build_independent_verifier_context,
is_report_verified,
validate_evidence_transition,
verify_finding,
write_verification_results,
)
def test_evidence_transition_rejects_backward_and_skipped_levels() -> None:
assert validate_evidence_transition("suspicion", "static_corroboration") == EvidenceLevel.STATIC_CORROBORATION
with pytest.raises(ValueError, match="backward"):
validate_evidence_transition("root_cause_explained", "static_corroboration")
with pytest.raises(ValueError, match="skip"):
validate_evidence_transition("suspicion", "root_cause_explained")
def test_verified_status_requires_static_corroboration_and_acceptance() -> None:
assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED)
assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED)