From 67699e8737101bba7936b0860d8782429dc0344a Mon Sep 17 00:00:00 2001
From: Marius Ciepluch <11855163+norandom@users.noreply.github.com>
Date: Wed, 30 Sep 2026 08:35:41 +0200
Subject: [PATCH] bench(plane): increment gate met after the tie-break -- 16/20
 agreed at $0.0205/candidate

A third verify pass on the 10 disputed candidates, 2-of-3 agreement (Req 6.4),
run on ax by extending the same Run: 16/20 declared sites agreed (reference 16),
$0.0205 per candidate with recorded triage (reference $0.022). Caveats recorded:
a design change on the same set; 17 vs 19 sites flagged in any pass; three
non-declared pgadmin candidates promoted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
---
 .kiro/specs/ai-service-plane/tasks.md         |  2 +-
 benchmarks/independent/plane-increment-2.json | 65 +++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 benchmarks/independent/plane-increment-2.json

diff --git a/.kiro/specs/ai-service-plane/tasks.md b/.kiro/specs/ai-service-plane/tasks.md
index 91bfb27..a1b2dd8 100644
--- a/.kiro/specs/ai-service-plane/tasks.md
+++ b/.kiro/specs/ai-service-plane/tasks.md
@@ -130,7 +130,7 @@ mypy are clean, and the commit is gated on pytest's own exit code. Script-level
     if not met, revise the design before task 9.
   - _Requirements: 6.1, 6.2, 6.3_
 
-- [ ] 8.1 Tie-break pass and re-measurement
+- [x] 8.1 Tie-break pass and re-measurement
   - `verify` takes `OUSAST_INPUT_ONLY` (pass c on the disputed candidates only); `agree` takes
     `OUSAST_INPUT_PASS_C` and decides 2-of-3 where pass c asked, the a/b rule elsewhere (`metrics.tiebreak`).
   - `validation-46` gains `<case>-vc` and `<case>-final` per case, appended so the first 60 entries keep their
diff --git a/benchmarks/independent/plane-increment-2.json b/benchmarks/independent/plane-increment-2.json
new file mode 100644
index 0000000..b45acb7
--- /dev/null
+++ b/benchmarks/independent/plane-increment-2.json
@@ -0,0 +1,65 @@
+{
+  "measurement": "ai-service-plane increment re-measured after the design revisit (Requirement 6.4): passes a and b as recorded in plane-increment-1.json, plus a tie-break pass c on the 10 candidates a and b disputed; agreement 2-of-3. Executed on google/ax by rerunning the same Run (the 60 done tasks were reused, 30 new tasks ran).",
+  "date": "2026-09-30",
+  "run": {
+    "name": "validation-46",
+    "tasks": 90,
+    "status": "done",
+    "runner_image": "localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617"
+  },
+  "previous": "benchmarks/independent/plane-increment-1.json",
+  "gate": {
+    "declared_sites_agreed": {
+      "plane": 16,
+      "reference": 16,
+      "of": 20,
+      "required": ">= 16",
+      "met": true
+    },
+    "cost_per_candidate_with_recorded_triage_usd": {
+      "plane_over_46": 0.0205,
+      "plane_over_43_after_triage": 0.02195,
+      "reference_over_46": 0.022,
+      "required": "< 0.022",
+      "met": true,
+      "note": "the reference divides by the 46 candidates before triage; on that basis the plane is 7% cheaper, on the 43 after triage it is 0.2% under"
+    },
+    "result": "MET"
+  },
+  "detail": {
+    "tiebreak": {
+      "asked": 10,
+      "flipped_to_agreed": 5,
+      "declared_sites_recovered": [
+        "froxlor listing",
+        "froxlor listingCount"
+      ],
+      "declared_site_still_disputed": "froxlor validateIpAddresses (b only)",
+      "non_declared_flipped_to_agreed": [
+        "pgadmin enable_disable",
+        "pgadmin get_job_args_value",
+        "pgadmin run_job"
+      ]
+    },
+    "agreed_candidates": {
+      "two_pass": 23,
+      "two_of_three": 28
+    },
+    "declared_sites_flagged_in_any_pass": {
+      "plane": 17,
+      "reference": 19,
+      "of": 20,
+      "missed_by_every_plane_pass": [
+        "budibase read",
+        "fuxa getDaqValue"
+      ]
+    },
+    "usd": {
+      "plane": 0.8567,
+      "recorded_triage": 0.0872,
+      "with_triage": 0.9439,
+      "reference": 1.0123
+    }
+  },
+  "reading": "The plane on ax matches the script pipeline's pre-registered recall (16/20 agreed) at 7% lower cost, so Requirement 6.2 is met after the 6.4 revisit. Two caveats travel with the result: (1) the tie-break is a design change chosen after seeing the first measurement on the same validation set, so this compares plane with script like for like and qualifies nothing on an independent population; (2) the plane's passes flag fewer declared sites than the reference's (17 vs 19 in any pass), and 2-of-3 also promoted three non-declared pgadmin candidates, whose truth is unassessed (an earlier pgadmin spot check found 3 of 5 findings truly privileged). Precision is not part of this gate and remains the open problem."
+}