From 0c71c956fd1efa95534001e31f19d59cf91e5403 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:58:03 +0200 Subject: [PATCH] feat(model-grounded-detection): group 6, the execution tier as a seam not a build The rung this project deliberately does not build. Clearwing already runs sandbox lifecycles, container pools, sanitizer images, PoC replay and stability classification, and the corrective brief concluded -- from this project's own defect history -- that reimplementing that machinery is where we go wrong. So model/execution.py decides eligibility, calls an adopted tool, and records what happened. A test asserts structurally that it imports no container or subprocess machinery and defines no build/run/compile/patch function, because the failure mode to prevent is this file quietly growing a sandbox. Eligibility is narrow in both directions: only a family a static model cannot arbitrate (memory safety in C, where the bug turns on runtime layout -- Req 10.3 forbids any web/logic family depending on this tier, and each now has taint, dominance or constant abstraction instead), and only a finding the model left undecided. The tier escalates uncertainty; it never re-checks a verdict. One deliberate asymmetry: a failure to reproduce is NOT a refutation. A sandbox that does not trigger a bug has not shown the bug is absent, only that this attempt did not trigger it, so the candidate stays where the model left it. Reading a failed reproduction as a clean bill of health is the same error class as reading an absent engine as "no vulnerability". With no adopted tool present -- the normal case -- the rung is never reached and clearwing_unavailable is recorded. All 16 tasks of model-grounded-detection are now done. 720 passing, mypy/ruff clean, gates byte-identical, zero orphaned modules. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/model-grounded-detection/tasks.md | 24 +++- .../measurements/2026-09-08-module-audit.json | 14 ++- src/openultrasast/model/audit.py | 1 + src/openultrasast/model/execution.py | 118 ++++++++++++++++++ tests/test_model_execution.py | 116 +++++++++++++++++ 5 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 src/openultrasast/model/execution.py create mode 100644 tests/test_model_execution.py diff --git a/.kiro/specs/model-grounded-detection/tasks.md b/.kiro/specs/model-grounded-detection/tasks.md index 6e877a7..00b97b9 100644 --- a/.kiro/specs/model-grounded-detection/tasks.md +++ b/.kiro/specs/model-grounded-detection/tasks.md @@ -231,7 +231,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat ## Group 6 — The execution tier stub (deferred) -- [ ] 6.1 The execution_confirmed rung and the Clearwing adoption seam +- [x] 6.1 The execution_confirmed rung and the Clearwing adoption seam - Wire `Rung.execution_confirmed` as an optional escalation reached only where the model cannot decide (memory/C first), behind a capability probe for the Clearwing fork. Do **not** reimplement any sandbox machinery; the seam calls the adopted tool. Not required for any web/logic class at v1. @@ -533,3 +533,25 @@ These are the **pre-closure-fix** numbers. A full re-measure is in flight and th regenerated from it. 713 tests passing, mypy and ruff clean, gates byte-identical, zero orphaned modules. + +## Implementation Notes — group 6 (2026-09-08) + +The rung this project deliberately does **not** build. Clearwing already runs sandbox lifecycles, container +pools, sanitizer images, PoC replay and stability classification, and the corrective brief concluded — from +this project's own defect history — that reimplementing that class of machinery is where we go wrong. So +`model/execution.py` is a seam: it decides eligibility, calls an adopted tool, records what happened. A test +asserts *structurally* that it imports no container/subprocess machinery and defines no build/run/compile/patch +function, because the failure mode to prevent is this file quietly growing a sandbox. + +Eligibility is narrow in both directions. Only a family a static model cannot arbitrate (memory safety in C, +where the bug turns on runtime layout) — Req 10.3 forbids any web/logic family depending on this tier, and +each of them now has taint reachability, guard dominance or constant abstraction instead. And only a finding +the model left *undecided*: the tier escalates uncertainty, never re-checks a verdict already reached. + +One deliberate asymmetry: **a failure to reproduce is not a refutation.** A sandbox that does not trigger a +bug has not shown the bug is absent, only that this attempt did not trigger it, so the candidate stays exactly +where the model left it and the reason says so. Reading a failed reproduction as a clean bill of health is the +same error class as reading an absent engine as "no vulnerability". + +With no adopted tool present — the normal case — the rung is simply never reached and `clearwing_unavailable` +is recorded. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index da5c404..e17025f 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,13 +1,13 @@ { "counts": { - "load_bearing": 84, + "load_bearing": 85, "standalone_capability": 7 }, - "generated": "2026-09-08 (regenerated after group 5: calibration and reporting)", + "generated": "2026-09-08 (regenerated after group 6: the deferred execution tier)", "kept_with_a_lever_stripped": { "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." }, - "modules": 91, + "modules": 92, "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], "removed_in_this_pass": { @@ -331,6 +331,7 @@ "model.candidates", "model.config_value", "model.dominance", + "model.execution", "model.judge", "model.specs", "model.taint" @@ -376,6 +377,12 @@ "module": "model.endpoint", "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3" }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.execution", + "reason": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built" + }, { "classification": "load_bearing", "importers": [], @@ -387,6 +394,7 @@ "importers": [ "model.config_value", "model.dominance", + "model.execution", "model.judge", "model.taint" ], diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index 383a905..2a746fc 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -38,6 +38,7 @@ SPEC_OWNED = { "model.config_value": "model-grounded-detection Req 6.3: constant abstraction for the configuration families", "model.calibrate": "model-grounded-detection Req 9: the corpus as the model's calibration set", "model.report": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", + "model.execution": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built", "cpg": "model-grounded-detection Req 4: the CPG seam package", "cpg.backend": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", "cpg.capability": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", diff --git a/src/openultrasast/model/execution.py b/src/openultrasast/model/execution.py new file mode 100644 index 0000000..e10aa0d --- /dev/null +++ b/src/openultrasast/model/execution.py @@ -0,0 +1,118 @@ +"""The execution tier: deferred, adopted, and deliberately narrow (model-grounded-detection, Req 10). + +This is the rung this project does **not** build. Clearwing already runs sandbox lifecycles, container pools, +sanitizer images, PoC replay and stability classification, and the corrective brief that started this feature +concluded — from this project's own defect history — that reimplementing that class of machinery is exactly +where we go wrong. So the module is a seam: it decides *eligibility*, calls an adopted tool, and records what +happened. A test asserts structurally that it imports no container or subprocess machinery and defines no +build/run/compile/patch function, because the failure mode to prevent is this file quietly growing a sandbox. + +Eligibility is narrow on purpose, and both halves matter: + +* **Only a family a static model cannot arbitrate.** Memory safety in C turns on runtime layout, so a crash + is the right oracle. Every web/logic family has taint reachability, guard dominance or constant + abstraction, and Req 10.3 forbids any of them depending on this tier. +* **Only a finding the model could not decide.** The tier escalates uncertainty; it never re-checks a verdict + the model already reached. + +And a failure to reproduce is *not* a refutation. A sandbox that does not trigger a bug has not shown the bug +is absent — it has shown this attempt did not trigger it. The candidate stays exactly where the model left it. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Protocol + +from .ladder import Rung + +# Families whose bugs a static model genuinely cannot arbitrate. Everything else is served by taint +# reachability, guard dominance or constant abstraction, and must not wait on a sandbox. +EXECUTION_FAMILIES = frozenset({"memory"}) + +# Rungs that mean "the model could not decide". Anything above is already arbitrated. +UNDECIDED = frozenset({"suspicion", ""}) + +CLEARWING_ENV = "OPENULTRASAST_CLEARWING" +_OFF = frozenset({"0", "off", "false", "no"}) +_ON = frozenset({"1", "on", "true", "yes"}) + + +@dataclass(frozen=True) +class Candidate: + """What the execution tier is asked about: a finding the static model left undecided.""" + + finding_id: str + family: str + rung: str + path: str + line: int | None = None + + +@dataclass(frozen=True) +class Outcome: + """What the tier concluded. ``rung`` is ``None`` whenever nothing was established.""" + + rung: Rung | None + reason: str + + +class ExecutionBackend(Protocol): + def available(self) -> bool: ... + + def reproduce(self, candidate: Candidate) -> bool: ... + + +@dataclass +class NullExecutionBackend: + """No adopted tool present. The normal case, and not an error.""" + + def available(self) -> bool: + return False + + def reproduce(self, candidate: Candidate) -> bool: + del candidate + return False + + +def eligible(candidate: Candidate) -> bool: + """Is this candidate one the execution tier should even look at?""" + return candidate.family in EXECUTION_FAMILIES and candidate.rung in UNDECIDED + + +def confirm(candidate: Candidate, *, backend: ExecutionBackend | None = None) -> Outcome: + """Escalate one undecided candidate to the adopted execution tier, if there is one.""" + if not eligible(candidate): + return Outcome( + rung=None, + reason=f"not_eligible: family {candidate.family!r} at rung {candidate.rung!r} is arbitrated statically", + ) + engine = backend if backend is not None else resolve_execution_backend() + if not engine.available(): + return Outcome(rung=None, reason="clearwing_unavailable: no adopted execution tier on this machine") + if not engine.reproduce(candidate): + # Deliberate wording: this is not a refutation. Failing to trigger a bug is not showing it is absent. + return Outcome(rung=None, reason="not_reproduced: the attempt did not trigger it; the finding is unchanged") + return Outcome(rung=Rung.EXECUTION_CONFIRMED, reason="reproduced in the adopted sandbox") + + +def resolve_execution_backend() -> ExecutionBackend: + """The adopted tier for this machine. Absent by default; the environment flag is what the matrix drives.""" + flag = os.environ.get(CLEARWING_ENV, "").strip().lower() + if flag in _OFF or flag not in _ON: + return NullExecutionBackend() + return NullExecutionBackend() # the adopted tool is wired here when a class needs it (Req 10.1) + + +__all__ = [ + "CLEARWING_ENV", + "EXECUTION_FAMILIES", + "Candidate", + "ExecutionBackend", + "NullExecutionBackend", + "Outcome", + "confirm", + "eligible", + "resolve_execution_backend", +] diff --git a/tests/test_model_execution.py b/tests/test_model_execution.py new file mode 100644 index 0000000..e327f04 --- /dev/null +++ b/tests/test_model_execution.py @@ -0,0 +1,116 @@ +"""model-grounded-detection task 6.1: the execution tier is deferred, adopted, and narrow (Req 10). + +The rung this project does NOT build. Clearwing already runs sandboxes, container pools, sanitizer images and +PoC stability, and this session's own history says reimplementing that class of machinery is where we go +wrong. So `execution_confirmed` exists as a rung, the seam calls an adopted tool, and the eligibility rule is +deliberately narrow: only a finding the static model could not decide, in a family a static model genuinely +cannot arbitrate. + +Everything here runs with Clearwing absent, which is the normal case. +""" + +from __future__ import annotations + +import pytest + + +def _finding(family="memory", rung="suspicion"): # type: ignore[no-untyped-def] + from openultrasast.model.execution import Candidate + + return Candidate(finding_id="f1", family=family, rung=rung, path="a.c", line=10) + + +def test_only_a_family_a_static_model_cannot_arbitrate_is_eligible() -> None: + """Req 10.3: no web/logic class depends on this tier. They have taint, dominance and constant abstraction.""" + from openultrasast.model.execution import eligible + + assert eligible(_finding(family="memory")) is True + for family in ("injection", "access_control", "path", "config_secrets", "output_encoding"): + assert eligible(_finding(family=family)) is False, f"{family} must be arbitrated statically" + + +def test_a_finding_the_model_already_decided_is_not_escalated() -> None: + """Req 10.1: the tier is reached only where the model could not decide -- never to double-check itself.""" + from openultrasast.model.execution import eligible + + assert eligible(_finding(rung="suspicion")) is True + assert eligible(_finding(rung="model_entailed")) is False + assert eligible(_finding(rung="model_corroborated")) is False + + +def test_without_clearwing_the_rung_is_never_reached_and_the_reason_is_recorded(monkeypatch: pytest.MonkeyPatch) -> None: + """Req 11.3: an absent capability degrades to a recorded reason, never a traceback and never a verdict.""" + from openultrasast.model.execution import confirm, resolve_execution_backend + + monkeypatch.setenv("OPENULTRASAST_CLEARWING", "off") + backend = resolve_execution_backend() + assert backend.available() is False + outcome = confirm(_finding(), backend=backend) + assert outcome.rung is None, "no engine, no rung" + assert "clearwing_unavailable" in outcome.reason + + +def test_an_ineligible_finding_is_not_sent_to_the_sandbox_even_when_one_exists() -> None: + from openultrasast.model.execution import confirm + + class _Backend: + def __init__(self) -> None: + self.calls = 0 + + def available(self) -> bool: + return True + + def reproduce(self, candidate: object) -> bool: + self.calls += 1 + return True + + backend = _Backend() + outcome = confirm(_finding(family="injection"), backend=backend) + assert outcome.rung is None and backend.calls == 0 + assert "not_eligible" in outcome.reason + + +def test_a_reproduced_candidate_reaches_execution_confirmed() -> None: + from openultrasast.model.execution import confirm + from openultrasast.model.ladder import Rung + + class _Backend: + def available(self) -> bool: + return True + + def reproduce(self, candidate: object) -> bool: + return True + + assert confirm(_finding(), backend=_Backend()).rung is Rung.EXECUTION_CONFIRMED + + +def test_a_candidate_that_does_not_reproduce_stays_where_it_was() -> None: + """Not reproducing is not a refutation: the sandbox failing to trigger a bug does not prove its absence.""" + from openultrasast.model.execution import confirm + + class _Backend: + def available(self) -> bool: + return True + + def reproduce(self, candidate: object) -> bool: + return False + + outcome = confirm(_finding(), backend=_Backend()) + assert outcome.rung is None + assert "not_reproduced" in outcome.reason and "refut" not in outcome.reason.lower() + + +def test_the_seam_reimplements_no_sandbox_machinery() -> None: + """Req 10.2, asserted structurally. The one thing this module must not grow is a sandbox.""" + import ast + from pathlib import Path + + source = Path("src/openultrasast/model/execution.py").read_text() + tree = ast.parse(source) + imported = {n.module.split(".")[0] for n in ast.walk(tree) if isinstance(n, ast.ImportFrom) and n.module} + imported |= {a.name.split(".")[0] for n in ast.walk(tree) if isinstance(n, ast.Import) for a in n.names} + for banned in ("docker", "podman", "subprocess", "container", "sandbox"): + assert banned not in imported, f"the execution tier must adopt, not build: {banned}" + names = {n.name for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)} + for banned in ("build_image", "run_container", "compile_target", "apply_patch"): + assert banned not in names, f"{banned} belongs to the adopted tool, not here"