search-with-proof: startup environment and data files, dead-end dedupe, task_spend continues

Demos may set a bounded environment (loader variables denied) and up to 8 small
data files under .demo/; the reason step sees conclusions and may not re-propose
them; a task spend limit concludes its intent, not the search; finish wins over
stray tool calls.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_search_coordinator.py

@@ -166,7 +166,7 @@ def test_invalid_identifiers_refused_before_paths(tmp_path):
             SearchTask(bad, "search-1", "reason", "", {}, asdict(SearchBudget()))
 
 
-@pytest.mark.parametrize("detail", ["task_spend", "worker_steps", "worker_wall", "run_ceiling"])
+@pytest.mark.parametrize("detail", ["worker_steps", "worker_wall", "run_ceiling"])
 def test_worker_limit_preserves_fact_and_survives_resume(tmp_path, detail):
     def execute(task):
         if task.step == "reason":
@@ -217,3 +217,95 @@ def test_coordinator_identifies_each_own_limit(tmp_path, detail):
     ).run()
     assert state["end_reason"] == "budget_spent"
     assert state["end_detail"] == detail
+
+
+def test_duplicate_direction_retried_once_with_conclusions_and_priority(tmp_path):
+    import yaml
+
+    calls = []
+
+    def execute(task):
+        calls.append(task)
+        if len(calls) == 1:
+            return {"status": "ok", "cost_usd": 0, "intents": [{"description": "inspect blocked jwt startup configuration"}]}
+        if task.step == "explore":
+            return {"status": "execution_failure", "cost_usd": 0, "failure": "no startup keys available"}
+        view = yaml.safe_load(task.snapshot)
+        assert view["intents"][0]["conclusion"] == {"failure": "no startup keys available"}
+        if len(calls) == 4:
+            assert view["Priority"] == "construct the demo now with the facts available, or finish with the precise blocker."
+            assert view["rejections"][0]["reason"] == "duplicate_intent"
+        return {"status": "ok", "cost_usd": 0, "intents": [{"description": "inspect blocked jwt startup configuration again"}]}
+
+    state = Coordinator(board(tmp_path), execute).run()
+    assert [t.step for t in calls] == ["reason", "explore", "reason", "reason"]
+    assert len(state["intents"]) == 1
+    assert [f["reason"] for f in state["checkpoint"]["failures"]] == ["duplicate_intent"] * 2
+
+
+@pytest.mark.parametrize("with_fact", [False, True])
+def test_task_spend_concludes_and_continues_to_demo(tmp_path, with_fact):
+    import yaml
+
+    calls = []
+
+    def execute(task):
+        calls.append(task)
+        if len(calls) == 1:
+            return {"status": "ok", "cost_usd": 0, "intents": [{"description": "inspect application route"}]}
+        if len(calls) == 2:
+            result = {
+                "status": "execution_failure",
+                "failure": "allowance used",
+                "cost_usd": 0.01,
+                "budget_exhaustion": {"end_detail": "task_spend"},
+            }
+            if with_fact:
+                result.update(status="ok", fact={"text": "route confirmed", "evidence_refs": ["tool:1"]})
+            return result
+        if task.step == "reason":
+            view = yaml.safe_load(task.snapshot)
+            concluded = view["intents"][0]
+            assert concluded["status"] == "concluded" and concluded["limit"] == "task_spend"
+            assert concluded["conclusion"] == ({"facts": ["fact-task-2"]} if with_fact else {"failure": "allowance used"})
+            return {"status": "ok", "cost_usd": 0, "intents": [{"description": "construct runnable demo"}]}
+        if task.step == "explore":
+            return {"status": "ok", "cost_usd": 0, "fact": {"text": "demo", "evidence_refs": ["tool:2"], "demo": "demo/"}}
+        return {"status": "ok", "cost_usd": 0, "outcome": "demonstrated", "evidence_refs": ["verify:1"]}
+
+    state = Coordinator(board(tmp_path), execute).run()
+    assert state["end_reason"] == "goal_met"
+    assert [t.step for t in calls] == ["reason", "explore", "reason", "explore", "verify"]
+    assert len(state["facts"]) == (3 if with_fact else 2)
+
+
+@pytest.mark.parametrize(
+    "description,rejected",
+    [
+        ("READ blocked route plus fresh", False),  # exactly 60%, case-insensitive
+        ("READ blocked route plus", True),
+        ("read blocked route", True),
+    ],
+)
+def test_duplicate_word_overlap_threshold(tmp_path, description, rejected):
+    coordinator = Coordinator(board(tmp_path), lambda task: None)
+    coordinator.state["intents"] = [
+        {
+            "id": "old",
+            "description": "read blocked route",
+            "status": "concluded",
+            "failure": "unavailable",
+            "from_facts": [],
+            "step": "reason",
+            "task_id": "old-task",
+        }
+    ]
+    added, duplicate = coordinator._admit_intents(
+        {
+            "status": "ok",
+            "task_id": "new-task",
+            "intents": [{"description": description}],
+        }
+    )
+    assert duplicate is rejected
+    assert added == (0 if rejected else 1)

tests/test_search_demo.py

@@ -123,3 +123,105 @@ def test_recipe_rejects_other_ecosystem(tmp_path, recipe, manifest, directory):
     value["build"] = {"recipe": recipe, "arguments": ["." if directory else manifest]}
     with pytest.raises(ValueError, match="ecosystem"):
         validate_demo(value, checkout=tmp_path)
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "LD_PRELOAD",
+        "LD_LIBRARY_PATH",
+        "PYTHONPATH",
+        "PYTHONSTARTUP",
+        "NODE_OPTIONS",
+        "NODE_PATH",
+        "JAVA_TOOL_OPTIONS",
+        "_JAVA_OPTIONS",
+        "JDK_JAVA_OPTIONS",
+        "PHP_INI_SCAN_DIR",
+        "PHPRC",
+        "PATH",
+        "HOME",
+        "TMPDIR",
+        "OUSAST_SECRET",
+        "MY_CANARY_KEY",
+        "lowercase",
+        "1KEY",
+        "A" * 65,
+    ],
+)
+def test_environment_denylist(name):
+    value = document()
+    value["start"]["environment"] = {name: "value"}
+    with pytest.raises(ValueError):
+        validate_demo(value)
+
+
+@pytest.mark.parametrize("content", ["${canary}", "/fixture/canary", "$PROOF_MARKER", "x\0y"])
+@pytest.mark.parametrize("field,key", [("environment", "JWT_KEY"), ("files", "jwt.key")])
+def test_config_preserves_no_canary_rule(field, key, content):
+    value = document()
+    value["start"][field] = {key: content}
+    with pytest.raises(ValueError):
+        validate_demo(value)
+
+
+def test_config_boundaries_and_private_file_modes(tmp_path):
+    from openultrasast.search.demo import write_start_files
+
+    value = document()
+    value["start"]["environment"] = {f"KEY_{i}": "é" * 4096 for i in range(32)}
+    value["start"]["files"] = {f"keys/{i}.pem": "é" * 32768 for i in range(8)}
+    assert validate_demo(value) == value
+    write_start_files(tmp_path, value["start"])
+    for name, content in value["start"]["files"].items():
+        path = tmp_path / ".demo" / name
+        assert path.read_text() == content
+        assert path.stat().st_mode & 0o777 == 0o600
+    for field, key, content in [("environment", "EXTRA", "x"), ("files", "extra.pem", "x")]:
+        value["start"][field][key] = content
+        with pytest.raises(ValueError):
+            validate_demo(value)
+        del value["start"][field][key]
+    value["start"]["files"]["keys/0.pem"] += "x"
+    with pytest.raises(ValueError):
+        validate_demo(value)
+    value["start"]["files"] = {}
+    value["start"]["environment"]["KEY_0"] += "x"
+    with pytest.raises(ValueError):
+        validate_demo(value)
+
+
+@pytest.mark.parametrize("name", ["../key.pem", "a/../key.pem", "/tmp/key.pem", "a..pem", "app.py", "run.sh", "x.exe", "a\\b.pem"])
+def test_data_file_paths_and_extensions(name):
+    value = document()
+    value["start"]["files"] = {name: "data"}
+    with pytest.raises(ValueError):
+        validate_demo(value)
+
+
+@pytest.mark.parametrize("where", ["root", "parent", "file", "existing"])
+def test_data_files_never_follow_links_or_overwrite(tmp_path, where):
+    from openultrasast.search.demo import write_start_files
+
+    outside = tmp_path / "outside"
+    outside.mkdir()
+    checkout = tmp_path / "checkout"
+    checkout.mkdir()
+    root = checkout / ".demo"
+    if where == "root":
+        root.symlink_to(outside, target_is_directory=True)
+    else:
+        root.mkdir()
+        if where == "parent":
+            (root / "keys").symlink_to(outside, target_is_directory=True)
+        else:
+            (root / "keys").mkdir()
+            if where == "file":
+                (root / "keys/key.pem").symlink_to(outside / "key.pem")
+            else:
+                (root / "keys/key.pem").write_text("original")
+    with pytest.raises((ValueError, FileExistsError)):
+        write_start_files(checkout, {"files": {"keys/key.pem": "changed"}})
+    assert not list(outside.iterdir())
+    if where == "existing":
+        assert (root / "keys/key.pem").read_text() == "original"

tests/test_search_task_storage.py

@@ -420,3 +420,37 @@ def test_command_failure_sanitizes_tokens_and_caps_tail(secret):
     failure = task_storage.CommandFailure("build", 1, "old\n" * 50 + "x" * 2000 + "\n" + secret)
     assert "hidden" not in failure.stderr and "old" not in failure.stderr
     assert len(failure.stderr) <= 1500
+
+
+def test_prepare_preserves_large_config_and_task_verifier_applies_it(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    source = (
+        "import os\nfrom pathlib import Path\n"
+        "assert os.environ['JWT_KEY'] == 'required-key'\n"
+        "assert Path(os.environ['JWT_FILE']).read_text() == 'x' * 65536\n"
+        "print('ready')\n"
+    )
+    (repo / "app.py").write_text(source)
+    request = spec()
+    request["demo"]["start"].update(
+        environment={"JWT_KEY": "required-key", "JWT_FILE": ".demo/jwt.pem"},
+        files={"jwt.pem": "x" * 65536, "other.json": "{}"},
+    )
+    blobs = []
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "ok", result
+    destination = workspace / "unpacked"
+    verify_task.extract_checkout(blobs[0], destination)
+    exported = json.loads((destination / "spec.json").read_text())
+    assert exported == request
+    assert not (destination / "checkout/.demo").exists()
+    demo = workspace / "demo"
+    demo.mkdir()
+    (demo / "demo.json").write_text(json.dumps(exported["demo"]))
+    observation = verify_side_task(Side(destination / "checkout", products=destination / "products"), demo, "path")
+    assert observation.outcome == "observed", observation
+    assert len(observation.runs) == 1

tests/test_search_verify.py

@@ -514,3 +514,30 @@ def test_xss_without_browser_is_unavailable_not_no_oracle(pair):
     edit(demo, lambda value: value.update(oracle="xss"))
     result = verify(*sides, demo, "output_encoding")
     assert result.outcome == "inconclusive" and "runtime unavailable" in result.reason
+
+
+def test_app_requires_environment_and_data_files_on_both_sides(pair):
+    sides, demo = pair
+    for side in sides:
+        path = side.checkout / "app.py"
+        source = path.read_text()
+        startup = (
+            "import os\nfrom pathlib import Path\n"
+            "assert os.environ['JWT_SECRET'] == 'local-test-key'\n"
+            "assert Path(os.environ['JWT_KEY_FILE']).read_text() == 'local-key-data'\n"
+        )
+        path.write_text(startup + source)
+    missing = verify(*sides, demo, "path")
+    assert missing.outcome == "inconclusive", missing
+    assert [side.outcome for side in missing.sides] == ["could_not_run", "could_not_run"]
+    edit(
+        demo,
+        lambda value: value["start"].update(
+            environment={"JWT_SECRET": "local-test-key", "JWT_KEY_FILE": ".demo/keys/jwt.pem"},
+            files={"keys/jwt.pem": "local-key-data"},
+        ),
+    )
+    result = verify(*sides, demo, "path")
+    assert result.outcome == "demonstrated", result
+    assert all(len(side.runs) == 3 for side in result.sides)
+    assert all(not (side.checkout / ".demo").exists() for side in sides)

tests/test_search_worker.py

@@ -443,3 +443,17 @@ def test_worker_run_uses_search_memory_budget(tmp_path):
     assert worker.tool("grep", {"pattern": "needle"})["memory_bytes"] == 256 * 1024**2
     worker.limits["memory_bytes"] = 128 * 1024**2
     assert worker.tool("read_file", {"path": "app.py"})["memory_bytes"] == 128 * 1024**2
+
+
+@pytest.mark.parametrize("finish_first", [False, True])
+def test_finish_wins_over_other_calls_without_executing_them(tmp_path, finish_first):
+    finish = tool("finish", fact={"text": "Confirmed app source", "evidence_refs": ["tool:1"]})["tool_calls"][0]
+    other = tool("read_file", path="must-not-read")["tool_calls"][0]
+    model = StubModel([tool("read_file", path="app.py"), {"tool_calls": [finish, other] if finish_first else [other, finish]}])
+    (tmp_path / "app.py").write_text("print('readable')")
+    worker = Worker(model, repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
+    result = worker(task("explore"))
+    assert result["status"] == "ok"
+    assert result["fact"]["text"] == "Confirmed app source"
+    assert worker.metrics["warnings"] == 1
+    assert worker.evidence == {"tool:1"}