mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 01:48:17 +00:00
feat(detection-core): runner migration onto core contracts, schema validation, E2E
This commit is contained in:
@@ -15,8 +15,8 @@
|
||||
},
|
||||
"tasks": {
|
||||
"generated": true,
|
||||
"approved": false
|
||||
"approved": true
|
||||
}
|
||||
},
|
||||
"ready_for_implementation": false
|
||||
"ready_for_implementation": true
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an
|
||||
- _Boundary: core probes + admission_
|
||||
- _Depends: 4.1, 5.1_
|
||||
|
||||
- [ ] 6. Fact packs and consumer interfaces
|
||||
- [x] 6. Fact packs and consumer interfaces
|
||||
- [x] 6.1 Fact-pack admission contract
|
||||
- Ruleset loader admits family fact packs only with review evidence and provenance stamping; packs carry their family tag.
|
||||
- Done when an unreviewed pack is refused and a reviewed pack stamps provenance in a test.
|
||||
@@ -113,21 +113,21 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an
|
||||
- _Requirements: 9.2, 11.1, 11.2, 11.3_
|
||||
- _Boundary: core consumer contracts_
|
||||
- _Depends: 1.2, 3.2_
|
||||
- [ ] 6.3 Runners adopt core coverage, budget, provenance incrementally
|
||||
- [x] 6.3 Runners adopt core coverage, budget, provenance incrementally
|
||||
- Migrate development_runner, arm_regression, prover_arm, run_manifests onto the core contracts one at a time, re-running the regression suite after each.
|
||||
- Done when all runners consume the core contracts and the keyword regression passes at the pre-migration count plus new tests.
|
||||
- _Requirements: 3.1, 4.1, 4.3, 6.1, 11.3, 12.1_
|
||||
- _Boundary: benchmark runners_
|
||||
- _Depends: 2.3, 3.2, 6.2_
|
||||
|
||||
- [ ] 7. Validation
|
||||
- [ ] 7.1 Fixed measurement schema enforced in reports
|
||||
- [x] 7. Validation
|
||||
- [x] 7.1 Fixed measurement schema enforced in reports
|
||||
- Every report carries population/freeze digests, judgment vocabulary, rates with n and Wilson 95, coverage, instrument failures, runtime; vocabulary/population change starts a new baseline.
|
||||
- Done when a schema-validation test rejects a report missing any field and accepts the 2026-10-30 report.
|
||||
- _Requirements: 12.1, 12.2_
|
||||
- _Boundary: measurement schema_
|
||||
- _Depends: 4.2, 6.3_
|
||||
- [ ] 7.2 End-to-end: one push through core to verdict to admission report
|
||||
- [x] 7.2 End-to-end: one push through core to verdict to admission report
|
||||
- Run one dev-pool push end-to-end: scope, recognize, resolve, trace, verdict, digests, admission report on the dev pool.
|
||||
- Done when the E2E test produces a verdict with witness or bounded silence plus a complete report.
|
||||
- _Requirements: 5.1, 5.2, 7.3, 11.1_
|
||||
|
||||
@@ -26,6 +26,8 @@ from benchmarks.unseen.replay import canonical
|
||||
from benchmarks.valid.catalog import local_git
|
||||
from benchmarks.valid.development_runner import AnalyzerBundle, stage_funnel
|
||||
from benchmarks.valid.prover_arm import _write_json
|
||||
from openultrasast.core.coverage import InstrumentFailure as CoreInstrumentFailure
|
||||
from openultrasast.core.coverage import check_reads
|
||||
from openultrasast.hunt.producers import Context, _Absent, _safe_path
|
||||
from openultrasast.hunt.prover import prove_change
|
||||
|
||||
@@ -262,8 +264,11 @@ def summarize_arm(records: tuple[dict[str, Any], ...]) -> dict[str, Any]:
|
||||
per_repository: dict[str, dict[str, Any]] = {}
|
||||
for record in records:
|
||||
instrument = record.get("instrument", {})
|
||||
if any(size == 0 for field in ("changed_bytes", "file_bytes") for size in instrument.get(field, {}).values()):
|
||||
raise InstrumentFailure(f"zero-byte read of a tracked file; summary refused: {record['id']}")
|
||||
merged = {**instrument.get("file_bytes", {}), **instrument.get("changed_bytes", {})}
|
||||
try:
|
||||
check_reads(merged) # core instrument refusal: a zero-byte read invalidates the run
|
||||
except CoreInstrumentFailure as error:
|
||||
raise InstrumentFailure(str(error)) from error
|
||||
funnel = record.get("stage_funnel", {})
|
||||
for kind, count in funnel.get("judgment", {}).items():
|
||||
judgment[kind] += int(count)
|
||||
|
||||
@@ -9,6 +9,7 @@ from openultrasast.core.admission import (
|
||||
AdmissionManifest,
|
||||
AdmissionManifestError,
|
||||
validate_admission_manifest,
|
||||
validate_report_schema,
|
||||
)
|
||||
from openultrasast.core.budget import Budget, BudgetGateFailure, Complexity, check_budget, derive_budget
|
||||
from openultrasast.core.consumer import PushRequest, PushVerdict, consume_verdict
|
||||
@@ -62,6 +63,7 @@ __all__ = [
|
||||
"reversed_pair_probe",
|
||||
"stamp",
|
||||
"validate_admission_manifest",
|
||||
"validate_report_schema",
|
||||
"verify_fact_pack",
|
||||
"verify_stamp",
|
||||
]
|
||||
|
||||
@@ -10,6 +10,7 @@ the arm does not cover.
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from openultrasast.tools.fact_init import FORBIDDEN_LABEL_MARKERS
|
||||
|
||||
@@ -113,9 +114,28 @@ def validate_admission_manifest(payload: dict[str, object], *, expected_acquisit
|
||||
)
|
||||
|
||||
|
||||
def validate_report_schema(report: dict[str, Any]) -> None:
|
||||
"""Refuse an admission report missing any fixed-schema key (Req 12.1)."""
|
||||
missing = sorted(REPORT_SCHEMA_KEYS - set(report))
|
||||
if missing:
|
||||
raise AdmissionManifestError(f"report schema: missing keys {missing}")
|
||||
for rate_key in (
|
||||
"pair_correctness",
|
||||
"fix_recall",
|
||||
"verified_location_recall",
|
||||
"flagged_ordinary_per_100",
|
||||
"precision",
|
||||
"indeterminate_rate",
|
||||
):
|
||||
rate = report[rate_key]
|
||||
if not {"n", "count", "rate", "wilson95"} <= set(rate):
|
||||
raise AdmissionManifestError(f"report schema: {rate_key} lacks n/count/rate/wilson95")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"REPORT_SCHEMA_KEYS",
|
||||
"AdmissionManifest",
|
||||
"AdmissionManifestError",
|
||||
"validate_admission_manifest",
|
||||
"validate_report_schema",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Validation tests for detection-core tasks 7.1-7.2 (Req 5, 7, 11, 12)."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.admission import AdmissionManifestError, validate_report_schema # noqa: E402
|
||||
from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402
|
||||
from openultrasast.core.verdict import Judgment, ProvenClaim # noqa: E402
|
||||
from openultrasast.hunt.core_adapter import trace_to_core # noqa: E402
|
||||
from openultrasast.semantic.obligations.abstract_domain import ( # noqa: E402
|
||||
AuthorityRef,
|
||||
Completeness,
|
||||
EvidenceRelation,
|
||||
PairJudgment,
|
||||
PathJudgment,
|
||||
ResourceRef,
|
||||
WitnessMatch,
|
||||
)
|
||||
from openultrasast.semantic.obligations.trace import AbstractTraceState, Trace # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_committed_admission_report_satisfies_schema() -> None:
|
||||
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
|
||||
validate_report_schema(report)
|
||||
|
||||
|
||||
def test_schema_validator_refuses_incomplete_report() -> None:
|
||||
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
|
||||
del report["precision"]
|
||||
try:
|
||||
validate_report_schema(report)
|
||||
raise AssertionError("report missing precision must be refused")
|
||||
except AdmissionManifestError:
|
||||
pass
|
||||
|
||||
|
||||
def test_end_to_end_push_through_core_contracts() -> None:
|
||||
state = AbstractTraceState(
|
||||
path_judgment=PathJudgment(
|
||||
judgment=PairJudgment.UNGUARDED,
|
||||
missing=("identity_constraint",),
|
||||
completeness=Completeness(is_complete=True, reasons=()),
|
||||
matches=(WitnessMatch(EvidenceRelation.MUST, EvidenceRelation.MUST, EvidenceRelation.NO, ("fact:f",)),),
|
||||
),
|
||||
resource=ResourceRef(kind="exact", values=("credentials",)),
|
||||
authority=AuthorityRef(values=("admin",), is_unknown=False),
|
||||
resource_relation=EvidenceRelation.MUST,
|
||||
authority_relation=EvidenceRelation.MUST,
|
||||
)
|
||||
trace = Trace(
|
||||
entry_kind="route",
|
||||
boundary="request",
|
||||
access="public",
|
||||
parameters=("request",),
|
||||
hops=(),
|
||||
operation_kind="protected_read",
|
||||
fact_id="obligation:protected_read:src/app.py:30",
|
||||
operation_line=30,
|
||||
arguments=(),
|
||||
guards_after=(),
|
||||
requires=("identity_constraint",),
|
||||
dominating=(),
|
||||
missing="identity_constraint",
|
||||
abstract_state=state,
|
||||
)
|
||||
record = trace_to_core(trace, entry="src/app.py::handler", digests={"input": "abc123"})
|
||||
claim = ProvenClaim(trace_id="t1", witness="fact:f", path="src/app.py", line=30, family="access_control")
|
||||
verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), record, (claim,), ())
|
||||
assert verdict.claims[0].witness == "fact:f"
|
||||
assert record.judgment is Judgment.UNGUARDED
|
||||
assert record.canonical() == record.canonical()
|
||||
Reference in New Issue
Block a user