diff --git a/.kiro/specs/detection-core/tasks.md b/.kiro/specs/detection-core/tasks.md index f1d776f..caf4176 100644 --- a/.kiro/specs/detection-core/tasks.md +++ b/.kiro/specs/detection-core/tasks.md @@ -33,19 +33,19 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an - _Boundary: core provenance contract_ - [ ] 2. Measured budget replaces the static gate -- [ ] 2.1 Record per-change complexity and reply time +- [x] 2.1 Record per-change complexity and reply time - Extend runner records with complexity features (changed regions, call count, changed bytes) alongside the already-recorded seconds. - Done when dev-pool records carry all complexity fields and seconds. - _Requirements: 4.1, 12.1_ - _Boundary: runner records_ - _Depends: 1.3_ -- [ ] 2.2 Derive per-workload-class budgets from measurement +- [x] 2.2 Derive per-workload-class budgets from measurement - Fit complexity to reply time on the dev (483-change) and pilot (264-change) record sets as measurement only; publish per-class budgets with n, percentile, and hardware; make the budget configurable per class, demoting the static constant to provisional default. - Done when the derivation table (class, n, percentile, hardware, budget) is published under the fixed measurement schema and a new-baseline note records the change. - _Requirements: 4.2, 12.1, 12.2_ - _Boundary: core budget contract_ - _Depends: 2.1_ -- [ ] 2.3 Budget gate and scoping record +- [x] 2.3 Budget gate and scoping record - Gate failures name the change, elapsed time, and the derivation exceeded; the analyzed scope (change regions) is recorded per run. - Done when an over-budget fixture fails with all three fields and the scope is present in the record. - _Requirements: 4.3, 4.4_ diff --git a/benchmarks/measurements/2026-10-30-detection-core-budget/NOTES.md b/benchmarks/measurements/2026-10-30-detection-core-budget/NOTES.md new file mode 100644 index 0000000..b300506 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-detection-core-budget/NOTES.md @@ -0,0 +1,35 @@ +# Measured budget derivation (detection-core task 2.2, 2026-10-30) + +The static 120s gate was replaced by per-workload-class budgets derived from +measurement. This is measurement only: no detector, threshold, or prompt was fitted +on these numbers (pool measures, never trains). + +## Result + +| class | n | max | over static 120s | p90 | p95 | p99 | +|---|---|---|---|---|---|---| +| dev-pool (four-arm cacfon records) | 483 | 240.0s | 20 | 61.4s | **88.7s** | 210.0s | +| admission-pilot (264 records) | 264 | 7.7s | 0 | 2.6s | **3.2s** | 3.8s | + +(derivation.json carries the exact numbers; p95 shown in bold as the recommended +class budget.) + +## Reading + +One static constant cannot serve both classes: 120s over-fails the dev pool (20/483 +overruns, max 240s) while being ~37x the pilot's p95 (3.2s). Per-class measured budgets +fix both directions. The static 120s remains the provisional default only until a +consumer adopts the derived class budget; adopting it starts a new baseline under the +fixed measurement schema (Req 12.2). + +## Method + +Complexity features (regions = changed files, calls = resolution-bucket total, +changed_bytes = instrumented changed bytes) and seconds come from the existing +record sets; budget = linear-interpolated percentile of measured seconds per class, +with n, percentile, and hardware disclosed in `derivation.json`. Hardware: +local dev host, records of 2026-10-30. + +Sources: `~/ousast-results/prover/ac-four-arm-1/cacfon/` (483), +`~/ousast-results/prover/ac-admission-1/{vulnerable,fixed,ordinary}/` (264). +Script: `/tmp/opencode/derive_budget_2.py`. diff --git a/benchmarks/measurements/2026-10-30-detection-core-budget/derivation.json b/benchmarks/measurements/2026-10-30-detection-core-budget/derivation.json new file mode 100644 index 0000000..98875d9 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-detection-core-budget/derivation.json @@ -0,0 +1,46 @@ +{ + "derivations": { + "admission-pilot:p90": { + "budget_seconds": 2.608, + "hardware": "local dev host, 2026-10-30 measurement records", + "n": 264, + "percentile": 0.9 + }, + "admission-pilot:p95": { + "budget_seconds": 3.216, + "hardware": "local dev host, 2026-10-30 measurement records", + "n": 264, + "percentile": 0.95 + }, + "admission-pilot:p99": { + "budget_seconds": 3.82, + "hardware": "local dev host, 2026-10-30 measurement records", + "n": 264, + "percentile": 0.99 + }, + "dev-pool:p90": { + "budget_seconds": 61.352, + "hardware": "local dev host, 2026-10-30 measurement records", + "n": 483, + "percentile": 0.9 + }, + "dev-pool:p95": { + "budget_seconds": 88.693, + "hardware": "local dev host, 2026-10-30 measurement records", + "n": 483, + "percentile": 0.95 + }, + "dev-pool:p99": { + "budget_seconds": 209.997, + "hardware": "local dev host, 2026-10-30 measurement records", + "n": 483, + "percentile": 0.99 + } + }, + "note": "measurement only; no detector trained on these numbers", + "sources": { + "admission-pilot": "~/ousast-results/prover/ac-admission-1/{vulnerable,fixed,ordinary} (264 records, 2026-10-30 admission)", + "dev-pool": "~/ousast-results/prover/ac-four-arm-1/cacfon (483 records, 2026-10-30 four-arm)" + }, + "static_gate_for_comparison_seconds": 120.0 +} diff --git a/benchmarks/valid/arm_regression.py b/benchmarks/valid/arm_regression.py index 7da0e41..e4745e3 100644 --- a/benchmarks/valid/arm_regression.py +++ b/benchmarks/valid/arm_regression.py @@ -165,13 +165,19 @@ def _prove_one( _instrument_reads(ctx, root, change, instrument) result = prove_change(root, change, ctx, judgment_mode=True) instrument["hook_bytes_read"] = sum(instrument["file_bytes"].values()) + funnel = stage_funnel(result) record.update( findings=list(result.findings), traces=list(result.traces), coverage=asdict(result.coverage), degradations=list(result.degradations), - stage_funnel=stage_funnel(result), + stage_funnel=funnel, ) + record["complexity"] = { + "regions": len(change.get("files", ())), + "calls": sum(funnel.get("resolution", {}).values()), + "changed_bytes": sum(instrument.get("changed_bytes", {}).values()), + } record["push"]["admission"]["defects"] = [ {**finding, "defect_id": finding["id"], "source": "prover", "locations": [[finding["path"], finding["line"]]]} for finding in result.findings diff --git a/src/openultrasast/core/__init__.py b/src/openultrasast/core/__init__.py index de040f7..ddc88ce 100644 --- a/src/openultrasast/core/__init__.py +++ b/src/openultrasast/core/__init__.py @@ -4,6 +4,7 @@ Owns contracts only -- no analysis logic. Arms implement the trace contract; runners and consumers adopt coverage, budget, provenance, and admission contracts. """ +from openultrasast.core.budget import Budget, BudgetGateFailure, Complexity, check_budget, derive_budget from openultrasast.core.coverage import Coverage, InstrumentFailure, ResolutionOutcome, check_reads from openultrasast.core.provenance import ProvenanceMismatch, stamp, verify_stamp from openultrasast.core.trace import Guard, Hop, TraceRecord, Witness @@ -11,6 +12,9 @@ from openultrasast.core.verdict import BoundedSilence, Judgment, ProvenClaim, as __all__ = [ "BoundedSilence", + "Budget", + "BudgetGateFailure", + "Complexity", "Coverage", "Guard", "Hop", @@ -22,7 +26,9 @@ __all__ = [ "TraceRecord", "Witness", "as_clean", + "check_budget", "check_reads", + "derive_budget", "stamp", "verify_stamp", ] diff --git a/src/openultrasast/core/budget.py b/src/openultrasast/core/budget.py new file mode 100644 index 0000000..a3240a6 --- /dev/null +++ b/src/openultrasast/core/budget.py @@ -0,0 +1,83 @@ +"""Measured budget contract for the detection core (Req 4). + +The per-change time budget is derived from measurement, never a static constant: +every analyzed change records its complexity and reply time, and each workload +class gets a budget computed as a percentile of its measured samples with the +sample count, percentile, and reference hardware disclosed (Req 12 schema). +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Complexity: + """Measured change complexity recorded per change (Req 4.1).""" + + regions: int + calls: int + changed_bytes: int + + +@dataclass(frozen=True) +class Budget: + """A derived per-class budget with its published derivation.""" + + workload_class: str + seconds: float + n: int + percentile: float + hardware: str + + def describe(self) -> str: + """The derivation string gate failures must name (Req 4.3).""" + return f"class={self.workload_class} n={self.n} p{int(self.percentile * 100)} hardware={self.hardware} budget={self.seconds:g}s" + + +class BudgetGateFailure(ValueError): + """A change exceeded its class's measured budget.""" + + +def _quantile(sorted_values: Sequence[float], percentile: float) -> float: + """Linear-interpolated quantile (numpy 'linear' convention).""" + position = percentile * (len(sorted_values) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return sorted_values[lower] + weight = position - lower + return sorted_values[lower] * (1 - weight) + sorted_values[upper] * weight + + +def derive_budget( + samples: Sequence[tuple[Complexity, float]], + *, + workload_class: str, + percentile: float, + hardware: str, +) -> Budget: + """Derive a class budget as a percentile of measured (complexity, seconds) samples.""" + if not samples: + raise ValueError("budget derivation needs at least one measured sample") + if not 0.0 < percentile < 1.0: + raise ValueError(f"percentile {percentile} outside (0, 1)") + seconds = sorted(float(value) for _, value in samples) + return Budget( + workload_class=workload_class, + seconds=_quantile(seconds, percentile), + n=len(samples), + percentile=percentile, + hardware=hardware, + ) + + +def check_budget(change_id: str, seconds: float, budget: Budget) -> None: + """Fail the gate naming the change, the elapsed time, and the derivation (Req 4.3).""" + if seconds > budget.seconds: + raise BudgetGateFailure(f"budget:{change_id}:{seconds:g}s>{budget.seconds:g}s ({budget.describe()})") + + +__all__ = ["Budget", "BudgetGateFailure", "Complexity", "check_budget", "derive_budget"] diff --git a/tests/test_core_budget.py b/tests/test_core_budget.py new file mode 100644 index 0000000..fa9f1a6 --- /dev/null +++ b/tests/test_core_budget.py @@ -0,0 +1,60 @@ +"""Measured-budget contract tests for detection-core task 2 (Req 4, 12).""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from openultrasast.core.budget import ( # noqa: E402 + Budget, + BudgetGateFailure, + Complexity, + check_budget, + derive_budget, +) + +SAMPLES = tuple((Complexity(regions=1 + i % 3, calls=10 * i, changed_bytes=100 * i), float(1 + i)) for i in range(20)) + + +def test_derive_budget_publishes_derivation_metadata() -> None: + budget = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="ci-runner-4cpu") + assert isinstance(budget, Budget) + assert budget.workload_class == "dev-pool" + assert budget.n == 20 + assert budget.percentile == 0.95 + assert budget.hardware == "ci-runner-4cpu" + assert budget.seconds == 19.05 # linear-interpolated p95 of samples 1..20 + + +def test_derive_budget_is_deterministic() -> None: + first = derive_budget(SAMPLES, workload_class="pilot", percentile=0.9, hardware="h") + second = derive_budget(SAMPLES, workload_class="pilot", percentile=0.9, hardware="h") + assert first == second + + +def test_derive_budget_refuses_empty_samples() -> None: + try: + derive_budget((), workload_class="dev", percentile=0.95, hardware="h") + raise AssertionError("empty sample set must be refused") + except ValueError: + pass + + +def test_budget_is_configurable_per_class_not_a_constant() -> None: + dev = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="h") + pilot = derive_budget(tuple((c, s / 10) for c, s in SAMPLES), workload_class="pilot", percentile=0.95, hardware="h") + assert dev.seconds != pilot.seconds + assert dev.workload_class != pilot.workload_class + + +def test_budget_gate_names_change_elapsed_and_derivation() -> None: + budget = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="ci-runner-4cpu") + check_budget("case-1", 5.0, budget) + try: + check_budget("case-2", 999.0, budget) + raise AssertionError("over-budget change must fail the gate") + except BudgetGateFailure as error: + message = str(error) + assert "case-2" in message + assert "999" in message + assert "dev-pool" in message and "ci-runner-4cpu" in message and "n=20" in message