From 4c356caa214664fff8242efc6668e2651b315f78 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:05:07 +0200 Subject: [PATCH] measurements: py re-validation 2 -- served-root fix works (demo now wires ${served_root}); still inconclusive, budget-bound [skip ci] Served-root guidance validated: authored demo now references ${served_root}. Still inconclusive (0 false proofs); agent spread budget across vectors and hit run_ceiling (164 calls, $1.47/$1.50) without converging, and the committed traversal did not differential. Remaining = per-app exploit-authoring + convergence (the layer the $0 local glance proof did by hand). Co-Authored-By: Claude Opus 4.8 --- .../record.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 benchmarks/measurements/2026-10-07-search-pilot-py-revalidate2/record.json diff --git a/benchmarks/measurements/2026-10-07-search-pilot-py-revalidate2/record.json b/benchmarks/measurements/2026-10-07-search-pilot-py-revalidate2/record.json new file mode 100644 index 0000000..abe3b6e --- /dev/null +++ b/benchmarks/measurements/2026-10-07-search-pilot-py-revalidate2/record.json @@ -0,0 +1,15 @@ +{ + "record": "2026-10-07 search-with-proof: Python pair re-validation after served-root-wiring guidance fix", + "ceiling_usd": 1.5, + "pair": {"index": 13, "language": "python", "family": "path"}, + "outcome": "inconclusive", + "end_detail": "run_ceiling", + "verify_tasks": 6, + "model_calls": 164, + "settled_usd": 1.4661, + "wall_seconds": 2025, + "false_proofs_fixed_side": 0, + "demo_wires_served_root": true, + "finding": "Served-root fix VALIDATED: the authored demo now references ${served_root} (it did not before). Still inconclusive, 0 false proofs. Two remaining issues: (1) the agent spread its budget across multiple disclosure vectors (the served-root traversal plus a separate temp-path approach) and hit the per-run ceiling (164 calls, $1.47 of $1.50) before converging on one working demo; (2) the committed demo's single traversal step did not produce a differential, so the exact traversal reaching the observable is not yet right. This is now the per-app exploit-authoring + convergence layer, the same layer the $0 local glance ceiling proof handled by a hand-authored demo.", + "progression": "parity -> target-aware; verify-time-only -> no explore trap; fair-share budget -> authors+verifies; served-root -> wires the served dir; remaining -> exact traversal + convergence within budget." +}