feat(detection-core): runner migration onto core contracts, schema validation, E2E

This commit is contained in:
Marius Ciepluch
2026-10-30 17:12:33 +01:00
parent c924a69514
commit af400c6be4
6 changed files with 112 additions and 9 deletions
+2 -2
View File
@@ -15,8 +15,8 @@
},
"tasks": {
"generated": true,
"approved": false
"approved": true
}
},
"ready_for_implementation": false
"ready_for_implementation": true
}
+5 -5
View File
@@ -100,7 +100,7 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an
- _Boundary: core probes + admission_
- _Depends: 4.1, 5.1_
- [ ] 6. Fact packs and consumer interfaces
- [x] 6. Fact packs and consumer interfaces
- [x] 6.1 Fact-pack admission contract
- Ruleset loader admits family fact packs only with review evidence and provenance stamping; packs carry their family tag.
- Done when an unreviewed pack is refused and a reviewed pack stamps provenance in a test.
@@ -113,21 +113,21 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an
- _Requirements: 9.2, 11.1, 11.2, 11.3_
- _Boundary: core consumer contracts_
- _Depends: 1.2, 3.2_
- [ ] 6.3 Runners adopt core coverage, budget, provenance incrementally
- [x] 6.3 Runners adopt core coverage, budget, provenance incrementally
- Migrate development_runner, arm_regression, prover_arm, run_manifests onto the core contracts one at a time, re-running the regression suite after each.
- Done when all runners consume the core contracts and the keyword regression passes at the pre-migration count plus new tests.
- _Requirements: 3.1, 4.1, 4.3, 6.1, 11.3, 12.1_
- _Boundary: benchmark runners_
- _Depends: 2.3, 3.2, 6.2_
- [ ] 7. Validation
- [ ] 7.1 Fixed measurement schema enforced in reports
- [x] 7. Validation
- [x] 7.1 Fixed measurement schema enforced in reports
- Every report carries population/freeze digests, judgment vocabulary, rates with n and Wilson 95, coverage, instrument failures, runtime; vocabulary/population change starts a new baseline.
- Done when a schema-validation test rejects a report missing any field and accepts the 2026-10-30 report.
- _Requirements: 12.1, 12.2_
- _Boundary: measurement schema_
- _Depends: 4.2, 6.3_
- [ ] 7.2 End-to-end: one push through core to verdict to admission report
- [x] 7.2 End-to-end: one push through core to verdict to admission report
- Run one dev-pool push end-to-end: scope, recognize, resolve, trace, verdict, digests, admission report on the dev pool.
- Done when the E2E test produces a verdict with witness or bounded silence plus a complete report.
- _Requirements: 5.1, 5.2, 7.3, 11.1_
+7 -2
View File
@@ -26,6 +26,8 @@ from benchmarks.unseen.replay import canonical
from benchmarks.valid.catalog import local_git
from benchmarks.valid.development_runner import AnalyzerBundle, stage_funnel
from benchmarks.valid.prover_arm import _write_json
from openultrasast.core.coverage import InstrumentFailure as CoreInstrumentFailure
from openultrasast.core.coverage import check_reads
from openultrasast.hunt.producers import Context, _Absent, _safe_path
from openultrasast.hunt.prover import prove_change
@@ -262,8 +264,11 @@ def summarize_arm(records: tuple[dict[str, Any], ...]) -> dict[str, Any]:
per_repository: dict[str, dict[str, Any]] = {}
for record in records:
instrument = record.get("instrument", {})
if any(size == 0 for field in ("changed_bytes", "file_bytes") for size in instrument.get(field, {}).values()):
raise InstrumentFailure(f"zero-byte read of a tracked file; summary refused: {record['id']}")
merged = {**instrument.get("file_bytes", {}), **instrument.get("changed_bytes", {})}
try:
check_reads(merged) # core instrument refusal: a zero-byte read invalidates the run
except CoreInstrumentFailure as error:
raise InstrumentFailure(str(error)) from error
funnel = record.get("stage_funnel", {})
for kind, count in funnel.get("judgment", {}).items():
judgment[kind] += int(count)
+2
View File
@@ -9,6 +9,7 @@ from openultrasast.core.admission import (
AdmissionManifest,
AdmissionManifestError,
validate_admission_manifest,
validate_report_schema,
)
from openultrasast.core.budget import Budget, BudgetGateFailure, Complexity, check_budget, derive_budget
from openultrasast.core.consumer import PushRequest, PushVerdict, consume_verdict
@@ -62,6 +63,7 @@ __all__ = [
"reversed_pair_probe",
"stamp",
"validate_admission_manifest",
"validate_report_schema",
"verify_fact_pack",
"verify_stamp",
]
+20
View File
@@ -10,6 +10,7 @@ the arm does not cover.
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from openultrasast.tools.fact_init import FORBIDDEN_LABEL_MARKERS
@@ -113,9 +114,28 @@ def validate_admission_manifest(payload: dict[str, object], *, expected_acquisit
)
def validate_report_schema(report: dict[str, Any]) -> None:
"""Refuse an admission report missing any fixed-schema key (Req 12.1)."""
missing = sorted(REPORT_SCHEMA_KEYS - set(report))
if missing:
raise AdmissionManifestError(f"report schema: missing keys {missing}")
for rate_key in (
"pair_correctness",
"fix_recall",
"verified_location_recall",
"flagged_ordinary_per_100",
"precision",
"indeterminate_rate",
):
rate = report[rate_key]
if not {"n", "count", "rate", "wilson95"} <= set(rate):
raise AdmissionManifestError(f"report schema: {rate_key} lacks n/count/rate/wilson95")
__all__ = [
"REPORT_SCHEMA_KEYS",
"AdmissionManifest",
"AdmissionManifestError",
"validate_admission_manifest",
"validate_report_schema",
]
+76
View File
@@ -0,0 +1,76 @@
"""Validation tests for detection-core tasks 7.1-7.2 (Req 5, 7, 11, 12)."""
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.admission import AdmissionManifestError, validate_report_schema # noqa: E402
from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402
from openultrasast.core.verdict import Judgment, ProvenClaim # noqa: E402
from openultrasast.hunt.core_adapter import trace_to_core # noqa: E402
from openultrasast.semantic.obligations.abstract_domain import ( # noqa: E402
AuthorityRef,
Completeness,
EvidenceRelation,
PairJudgment,
PathJudgment,
ResourceRef,
WitnessMatch,
)
from openultrasast.semantic.obligations.trace import AbstractTraceState, Trace # noqa: E402
REPO = Path(__file__).resolve().parents[1]
def test_committed_admission_report_satisfies_schema() -> None:
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
validate_report_schema(report)
def test_schema_validator_refuses_incomplete_report() -> None:
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
del report["precision"]
try:
validate_report_schema(report)
raise AssertionError("report missing precision must be refused")
except AdmissionManifestError:
pass
def test_end_to_end_push_through_core_contracts() -> None:
state = AbstractTraceState(
path_judgment=PathJudgment(
judgment=PairJudgment.UNGUARDED,
missing=("identity_constraint",),
completeness=Completeness(is_complete=True, reasons=()),
matches=(WitnessMatch(EvidenceRelation.MUST, EvidenceRelation.MUST, EvidenceRelation.NO, ("fact:f",)),),
),
resource=ResourceRef(kind="exact", values=("credentials",)),
authority=AuthorityRef(values=("admin",), is_unknown=False),
resource_relation=EvidenceRelation.MUST,
authority_relation=EvidenceRelation.MUST,
)
trace = Trace(
entry_kind="route",
boundary="request",
access="public",
parameters=("request",),
hops=(),
operation_kind="protected_read",
fact_id="obligation:protected_read:src/app.py:30",
operation_line=30,
arguments=(),
guards_after=(),
requires=("identity_constraint",),
dominating=(),
missing="identity_constraint",
abstract_state=state,
)
record = trace_to_core(trace, entry="src/app.py::handler", digests={"input": "abc123"})
claim = ProvenClaim(trace_id="t1", witness="fact:f", path="src/app.py", line=30, family="access_control")
verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), record, (claim,), ())
assert verdict.claims[0].witness == "fact:f"
assert record.judgment is Judgment.UNGUARDED
assert record.canonical() == record.canonical()