From 7c220307d40ec8b21dea6081729a052c9e474bbd Mon Sep 17 00:00:00 2001
From: Marius Ciepluch <11855163+norandom@users.noreply.github.com>
Date: Tue, 6 Oct 2026 19:35:42 +0200
Subject: [PATCH] measurements: path-family pilot after keystone +
 target-parity (0 demos, parity validated) [skip ci]

3 path searches (php/ts/py), vulnerable side, cluster verify lane, $2.95 of $3.
Payload-free aggregate; raw boards/demos stay gitignored. Finding: the parity fix
produced directed, target-aware search (the agent reasons about reaching the named
observable and inspects the app's serving guard), but no demonstration landed -- all
three spent their per-pair allowance before a differential, and two wrote zero verify
demos (budget consumed in reason/explore). Next levers: allowance/budget split toward
demo-writing, entrypoint discovery for php, and the explore-vs-verify path layout the
py agent flagged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .../2026-10-06-search-pilot-path/record.json  | 123 ++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 benchmarks/measurements/2026-10-06-search-pilot-path/record.json

diff --git a/benchmarks/measurements/2026-10-06-search-pilot-path/record.json b/benchmarks/measurements/2026-10-06-search-pilot-path/record.json
new file mode 100644
index 0000000..7bd08da
--- /dev/null
+++ b/benchmarks/measurements/2026-10-06-search-pilot-path/record.json
@@ -0,0 +1,123 @@
+{
+  "record": "2026-10-06 search-with-proof feasibility pilot (path family, vulnerable side, cluster verify lane)",
+  "image": "ghcr.io/norandom/ousast-search-task@sha256:60bfd73f...",
+  "note": "First pilot after the verify-side keystone and the agent-side path-oracle target-parity fix. Feasibility only; payloads/boards live in the gitignored private area.",
+  "ceiling_usd": 3.0,
+  "searches": [
+    {
+      "pair_index": 3,
+      "language": "php",
+      "family": "path",
+      "outcome": "inconclusive",
+      "end_reason": "budget_spent",
+      "end_detail": "demonstrations",
+      "model_calls": 78,
+      "executor_tasks": 12,
+      "verify_tasks": 6,
+      "settled_usd": 0.5779,
+      "wall_seconds": 439,
+      "failure_classes": [
+        {
+          "phase": "explore",
+          "class": "no cli entrypoint exists"
+        },
+        {
+          "phase": "verify/could_not_build",
+          "class": "runtime entry missing from"
+        },
+        {
+          "phase": "verify/could_not_build",
+          "class": "runtime entry missing from"
+        },
+        {
+          "phase": "verify",
+          "class": "noconsistentdifferential"
+        },
+        {
+          "phase": "verify/could_not_build",
+          "class": "runtime entry missing from"
+        },
+        {
+          "phase": "verify/could_not_build",
+          "class": "runtime entry missing from"
+        },
+        {
+          "phase": "verify",
+          "class": "noconsistentdifferential"
+        },
+        {
+          "phase": "verify/could_not_build",
+          "class": "runtime entry missing from"
+        },
+        {
+          "phase": "verify/could_not_build",
+          "class": "runtime entry missing from"
+        },
+        {
+          "phase": "verify",
+          "class": "noconsistentdifferential"
+        }
+      ]
+    },
+    {
+      "pair_index": 9,
+      "language": "typescript",
+      "family": "path",
+      "outcome": "plausible_unproven",
+      "end_reason": "budget_spent",
+      "end_detail": "reason_rounds",
+      "model_calls": 154,
+      "executor_tasks": 12,
+      "verify_tasks": 0,
+      "settled_usd": 1.1979,
+      "wall_seconds": 2300,
+      "failure_classes": []
+    },
+    {
+      "pair_index": 13,
+      "language": "python",
+      "family": "path",
+      "outcome": "plausible_unproven",
+      "end_reason": "budget_spent",
+      "end_detail": "run_ceiling",
+      "model_calls": 118,
+      "executor_tasks": 9,
+      "verify_tasks": 0,
+      "settled_usd": 1.1739,
+      "wall_seconds": 564,
+      "failure_classes": [
+        {
+          "phase": "explore",
+          "class": "as predicted the parent"
+        },
+        {
+          "phase": "explore",
+          "class": "passfalse canary workspacecheckoutcanary passfalse"
+        },
+        {
+          "phase": "explore",
+          "class": "with the servedroot prefix"
+        },
+        {
+          "phase": "explore",
+          "class": "h check at filepy"
+        },
+        {
+          "phase": "explore",
+          "class": "are plain attribute reads"
+        }
+      ]
+    }
+  ],
+  "aggregate": {
+    "outcomes": {
+      "inconclusive": 1,
+      "plausible_unproven": 2
+    },
+    "demonstrations": 0,
+    "false_proofs_fixed_side": 0,
+    "settled_usd": 2.9496,
+    "searches": 3
+  },
+  "finding": "Parity fix validated: the agent now runs directed, target-aware search toward the named observable (pair 13 reasoned explicitly about reaching it and inspected the app's file-serving guard). No demonstration: all three exhausted their per-pair allowance (demonstrations/reason_rounds/run_ceiling) before a differential; pairs 9 and 13 wrote 0 verify demos (budget consumed in reason/explore). Pair 3 (php) hit entrypoint-discovery failures (no CLI entrypoint / runtime entry missing). Cost ~$0.98/search, higher than the $0.2-0.5 estimate."
+}
\ No newline at end of file