From c5e6ed3221ea9fb682d8d8b604513e5195fd20c4 Mon Sep 17 00:00:00 2001
From: Marius Ciepluch <11855163+norandom@users.noreply.github.com>
Date: Tue, 29 Sep 2026 22:19:38 +0200
Subject: [PATCH] bench(plane): first increment measured on ax -- gate not met
 (14/20 agreed vs 16)

validation-46 ran done on google/ax: 60 tasks, 357 DeepSeek calls, $0.83 with
recorded triage ($0.0194 per candidate, reference $0.022: cost met). Declared
sites agreed 14/20 against the reference's 16/20: recall criterion not met.
Both runs find 19/20 sites in at least one pass; the two-pass agreement
flicker moved three sites. Design revisit before task 9.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
---
 .kiro/specs/ai-service-plane/tasks.md         |  2 +-
 benchmarks/independent/plane-increment-1.json | 37 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 benchmarks/independent/plane-increment-1.json

diff --git a/.kiro/specs/ai-service-plane/tasks.md b/.kiro/specs/ai-service-plane/tasks.md
index 0883454..3c61ba4 100644
--- a/.kiro/specs/ai-service-plane/tasks.md
+++ b/.kiro/specs/ai-service-plane/tasks.md
@@ -122,7 +122,7 @@ mypy are clean, and the commit is gated on pytest's own exit code. Script-level
   - Workspaces generated for the 15 validation cases' vulnerable pins; the runner image rebuilt and loaded.
   - _Requirements: 1.1, 1.2, 6.1_
 
-- [ ] 8. Measurement of the first increment
+- [x] 8. Measurement of the first increment
   - Run `validation-46` through the reconciler on ax (needs DeepSeek credit); record cost per candidate, tool turns
     per hunt, declared sites agreed, against `verifier-batched-check-2026-09-29.json`.
   - Run `benchmarks/dev/token_report.py` on the increment's development session.
diff --git a/benchmarks/independent/plane-increment-1.json b/benchmarks/independent/plane-increment-1.json
new file mode 100644
index 0000000..29314c5
--- /dev/null
+++ b/benchmarks/independent/plane-increment-1.json
@@ -0,0 +1,37 @@
+{
+  "measurement": "ai-service-plane first increment (Requirement 6): repo-facts + verify (2 passes) + agree for the 46-candidate validation set, executed by the ax-backed reconciler on google/ax + Agent Substrate (kind, gVisor) on the 7 GB host",
+  "date": "2026-09-29",
+  "run": {"name": "validation-46", "manifest": "plane/runs/validation-46.yaml", "tasks": 60, "workers": 2, "status": "done",
+          "results": "~/ousast-results/plane/validation-46/", "model": "deepseek-flash", "runner_image": "localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"},
+  "reference": "benchmarks/independent/verifier-batched-check-2026-09-29.json (script pipeline, same cases, candidates, pins and recorded triage)",
+  "gate": {
+    "declared_sites_agreed": {"plane": 14, "reference": 16, "of": 20, "required": ">= 16", "met": false},
+    "cost_per_candidate_with_recorded_triage_usd": {"plane": 0.0194, "plane_over_46": 0.0181, "reference": 0.022, "required": "< 0.022", "met": true},
+    "result": "NOT MET (recall criterion)"
+  },
+  "detail": {
+    "declared_sites_found_in_either_pass": {"plane": 19, "reference": 19, "of": 20},
+    "agreed_candidates": 23, "disputed_candidates": 10, "candidates_after_recorded_triage": 43,
+    "plane_usd": 0.7471, "recorded_triage_usd": 0.0872, "usd_with_triage": 0.8343, "reference_usd": 1.0123,
+    "calls": 357, "prompt_tokens": 3820395, "cache_hit_tokens": 2385152, "output_tokens": 62257,
+    "turns_per_hunt": 5.37, "hunts": 54,
+    "per_case_sites_agreed_vs_reference": {
+      "budibase-mongo-template-nosqli": "1/2 vs 2/2 (one site found in one pass only)",
+      "froxlor-admin-ip-second-order-sqli": "0/3 vs 2/3 (two sites found in one pass only)",
+      "unleash-addon-webhook-ssrf": "1/1 vs 0/1 (reference found it in one pass only)",
+      "fuxa-tdengine-escape-sqli": "0/1 vs 0/1", "winml-cli-serve-cors": "0/1 vs 0/1 (triage drops the only candidate)",
+      "all_other_cases": "equal"
+    }
+  },
+  "reading": "Detection is unchanged: both runs find 19 of 20 declared sites in at least one pass. The two-pass agreement rule's known flicker moved three sites into 'disputed' (net -2 against the reference). Cost fell 17% with the repo-facts callers line in the prompt and prompt caching (62% of prompt tokens were cache hits). By the pre-registered rule the increment is NOT MET and the design is revisited before task 9; the evidence points at the agreement rule (2-of-2 passes), not at the plane.",
+  "dev_token_report": {"transcript": "the whole development session (c09e30b2), not the increment alone", "unique_chars": "7.3M (~1.81M tokens)",
+                       "tool_use_input_pct": 45.4, "tool_result_pct": 40.1, "tool_io_pct": 85.5, "previous_tool_io_pct": 90,
+                       "tool_calls": 2701},
+  "live_defects_found_by_running_on_ax": [
+    "no Substrate WorkerPool", "tag-pinned images rejected", "ax CLI/server skew", "unknown-field rejection (atespace, commit)",
+    "server-set fields in AX_TASK_YAML", "tasks created Suspended; no Completed phase", "runner exited after the command",
+    "golden-snapshot boot ran the command", "egress gateway not deployed; deny-by-default per-actor policy",
+    "gateway dials the address the actor dialled", "workspace clone before the actor was running",
+    "32 KB env limit on inlined inputs", "63-byte template name limit"
+  ]
+}