From af400c6be4173c766bbb6216ea1add81ba143406 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:12:33 +0100 Subject: [PATCH] feat(detection-core): runner migration onto core contracts, schema validation, E2E --- .kiro/specs/detection-core/spec.json | 4 +- .kiro/specs/detection-core/tasks.md | 10 ++-- benchmarks/valid/arm_regression.py | 9 +++- src/openultrasast/core/__init__.py | 2 + src/openultrasast/core/admission.py | 20 ++++++++ tests/test_core_validation.py | 76 ++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 tests/test_core_validation.py diff --git a/.kiro/specs/detection-core/spec.json b/.kiro/specs/detection-core/spec.json index c5c7413..c08b02b 100644 --- a/.kiro/specs/detection-core/spec.json +++ b/.kiro/specs/detection-core/spec.json @@ -15,8 +15,8 @@ }, "tasks": { "generated": true, - "approved": false + "approved": true } }, - "ready_for_implementation": false + "ready_for_implementation": true } diff --git a/.kiro/specs/detection-core/tasks.md b/.kiro/specs/detection-core/tasks.md index 580bbc2..345a47f 100644 --- a/.kiro/specs/detection-core/tasks.md +++ b/.kiro/specs/detection-core/tasks.md @@ -100,7 +100,7 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an - _Boundary: core probes + admission_ - _Depends: 4.1, 5.1_ -- [ ] 6. Fact packs and consumer interfaces +- [x] 6. Fact packs and consumer interfaces - [x] 6.1 Fact-pack admission contract - Ruleset loader admits family fact packs only with review evidence and provenance stamping; packs carry their family tag. - Done when an unreviewed pack is refused and a reviewed pack stamps provenance in a test. @@ -113,21 +113,21 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an - _Requirements: 9.2, 11.1, 11.2, 11.3_ - _Boundary: core consumer contracts_ - _Depends: 1.2, 3.2_ -- [ ] 6.3 Runners adopt core coverage, budget, provenance incrementally +- [x] 6.3 Runners adopt core coverage, budget, provenance incrementally - Migrate development_runner, arm_regression, prover_arm, run_manifests onto the core contracts one at a time, re-running the regression suite after each. - Done when all runners consume the core contracts and the keyword regression passes at the pre-migration count plus new tests. - _Requirements: 3.1, 4.1, 4.3, 6.1, 11.3, 12.1_ - _Boundary: benchmark runners_ - _Depends: 2.3, 3.2, 6.2_ -- [ ] 7. Validation -- [ ] 7.1 Fixed measurement schema enforced in reports +- [x] 7. Validation +- [x] 7.1 Fixed measurement schema enforced in reports - Every report carries population/freeze digests, judgment vocabulary, rates with n and Wilson 95, coverage, instrument failures, runtime; vocabulary/population change starts a new baseline. - Done when a schema-validation test rejects a report missing any field and accepts the 2026-10-30 report. - _Requirements: 12.1, 12.2_ - _Boundary: measurement schema_ - _Depends: 4.2, 6.3_ -- [ ] 7.2 End-to-end: one push through core to verdict to admission report +- [x] 7.2 End-to-end: one push through core to verdict to admission report - Run one dev-pool push end-to-end: scope, recognize, resolve, trace, verdict, digests, admission report on the dev pool. - Done when the E2E test produces a verdict with witness or bounded silence plus a complete report. - _Requirements: 5.1, 5.2, 7.3, 11.1_ diff --git a/benchmarks/valid/arm_regression.py b/benchmarks/valid/arm_regression.py index e4745e3..fabc38a 100644 --- a/benchmarks/valid/arm_regression.py +++ b/benchmarks/valid/arm_regression.py @@ -26,6 +26,8 @@ from benchmarks.unseen.replay import canonical from benchmarks.valid.catalog import local_git from benchmarks.valid.development_runner import AnalyzerBundle, stage_funnel from benchmarks.valid.prover_arm import _write_json +from openultrasast.core.coverage import InstrumentFailure as CoreInstrumentFailure +from openultrasast.core.coverage import check_reads from openultrasast.hunt.producers import Context, _Absent, _safe_path from openultrasast.hunt.prover import prove_change @@ -262,8 +264,11 @@ def summarize_arm(records: tuple[dict[str, Any], ...]) -> dict[str, Any]: per_repository: dict[str, dict[str, Any]] = {} for record in records: instrument = record.get("instrument", {}) - if any(size == 0 for field in ("changed_bytes", "file_bytes") for size in instrument.get(field, {}).values()): - raise InstrumentFailure(f"zero-byte read of a tracked file; summary refused: {record['id']}") + merged = {**instrument.get("file_bytes", {}), **instrument.get("changed_bytes", {})} + try: + check_reads(merged) # core instrument refusal: a zero-byte read invalidates the run + except CoreInstrumentFailure as error: + raise InstrumentFailure(str(error)) from error funnel = record.get("stage_funnel", {}) for kind, count in funnel.get("judgment", {}).items(): judgment[kind] += int(count) diff --git a/src/openultrasast/core/__init__.py b/src/openultrasast/core/__init__.py index 5fefabf..5dc400a 100644 --- a/src/openultrasast/core/__init__.py +++ b/src/openultrasast/core/__init__.py @@ -9,6 +9,7 @@ from openultrasast.core.admission import ( AdmissionManifest, AdmissionManifestError, validate_admission_manifest, + validate_report_schema, ) from openultrasast.core.budget import Budget, BudgetGateFailure, Complexity, check_budget, derive_budget from openultrasast.core.consumer import PushRequest, PushVerdict, consume_verdict @@ -62,6 +63,7 @@ __all__ = [ "reversed_pair_probe", "stamp", "validate_admission_manifest", + "validate_report_schema", "verify_fact_pack", "verify_stamp", ] diff --git a/src/openultrasast/core/admission.py b/src/openultrasast/core/admission.py index 07ce867..f82e397 100644 --- a/src/openultrasast/core/admission.py +++ b/src/openultrasast/core/admission.py @@ -10,6 +10,7 @@ the arm does not cover. from __future__ import annotations from dataclasses import dataclass +from typing import Any from openultrasast.tools.fact_init import FORBIDDEN_LABEL_MARKERS @@ -113,9 +114,28 @@ def validate_admission_manifest(payload: dict[str, object], *, expected_acquisit ) +def validate_report_schema(report: dict[str, Any]) -> None: + """Refuse an admission report missing any fixed-schema key (Req 12.1).""" + missing = sorted(REPORT_SCHEMA_KEYS - set(report)) + if missing: + raise AdmissionManifestError(f"report schema: missing keys {missing}") + for rate_key in ( + "pair_correctness", + "fix_recall", + "verified_location_recall", + "flagged_ordinary_per_100", + "precision", + "indeterminate_rate", + ): + rate = report[rate_key] + if not {"n", "count", "rate", "wilson95"} <= set(rate): + raise AdmissionManifestError(f"report schema: {rate_key} lacks n/count/rate/wilson95") + + __all__ = [ "REPORT_SCHEMA_KEYS", "AdmissionManifest", "AdmissionManifestError", "validate_admission_manifest", + "validate_report_schema", ] diff --git a/tests/test_core_validation.py b/tests/test_core_validation.py new file mode 100644 index 0000000..f74256f --- /dev/null +++ b/tests/test_core_validation.py @@ -0,0 +1,76 @@ +"""Validation tests for detection-core tasks 7.1-7.2 (Req 5, 7, 11, 12).""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from openultrasast.core.admission import AdmissionManifestError, validate_report_schema # noqa: E402 +from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402 +from openultrasast.core.verdict import Judgment, ProvenClaim # noqa: E402 +from openultrasast.hunt.core_adapter import trace_to_core # noqa: E402 +from openultrasast.semantic.obligations.abstract_domain import ( # noqa: E402 + AuthorityRef, + Completeness, + EvidenceRelation, + PairJudgment, + PathJudgment, + ResourceRef, + WitnessMatch, +) +from openultrasast.semantic.obligations.trace import AbstractTraceState, Trace # noqa: E402 + +REPO = Path(__file__).resolve().parents[1] + + +def test_committed_admission_report_satisfies_schema() -> None: + report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8")) + validate_report_schema(report) + + +def test_schema_validator_refuses_incomplete_report() -> None: + report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8")) + del report["precision"] + try: + validate_report_schema(report) + raise AssertionError("report missing precision must be refused") + except AdmissionManifestError: + pass + + +def test_end_to_end_push_through_core_contracts() -> None: + state = AbstractTraceState( + path_judgment=PathJudgment( + judgment=PairJudgment.UNGUARDED, + missing=("identity_constraint",), + completeness=Completeness(is_complete=True, reasons=()), + matches=(WitnessMatch(EvidenceRelation.MUST, EvidenceRelation.MUST, EvidenceRelation.NO, ("fact:f",)),), + ), + resource=ResourceRef(kind="exact", values=("credentials",)), + authority=AuthorityRef(values=("admin",), is_unknown=False), + resource_relation=EvidenceRelation.MUST, + authority_relation=EvidenceRelation.MUST, + ) + trace = Trace( + entry_kind="route", + boundary="request", + access="public", + parameters=("request",), + hops=(), + operation_kind="protected_read", + fact_id="obligation:protected_read:src/app.py:30", + operation_line=30, + arguments=(), + guards_after=(), + requires=("identity_constraint",), + dominating=(), + missing="identity_constraint", + abstract_state=state, + ) + record = trace_to_core(trace, entry="src/app.py::handler", digests={"input": "abc123"}) + claim = ProvenClaim(trace_id="t1", witness="fact:f", path="src/app.py", line=30, family="access_control") + verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), record, (claim,), ()) + assert verdict.claims[0].witness == "fact:f" + assert record.judgment is Judgment.UNGUARDED + assert record.canonical() == record.canonical()