feat(contributor-scan): 1.3 the model layer reaches a user

`model-grounded-detection` finished 16/16 with the model layer having zero
production callers. This closes that. A real scan of a Flask file now reports:

  by_rung: {"model_entailed": 3, "model_corroborated": 0, "suspicion": 0}
  model:injection:ping:10  <- os.system("ping -c 1 " + request.args["host"])

The stage is the fifth MAP proposer, mirroring _check_obligations, and is
additive: pattern, overlay and obligation findings all keep working, and without
a CPG engine it records cpg_unavailable and changes nothing else. Two scan
integrity tests pin the exact degradation set and both broke -- correctly. Their
`requested == baseline` assertion, that findings are byte-identical, still
passed, so the only change is that the scan now SAYS the arbiter was absent
instead of being silently without one. The guards are updated to assert the
model degradation specifically rather than being loosened.

ModelLayerConfig, not ModelConfig: that name is already the [models] block, and
the type checker caught the collision immediately. Two meanings of one name is
how a scan silently reads the wrong settings.

Every payload carries the same shape, skipped ones included. A payload that is
sometimes a full record and sometimes a single "skipped" key is how a report
ends up quietly omitting a section.

unjudged_sample names the lowest-ranked regions the budget did not reach.
Counting them is the honesty requirement; naming them is what lets a maintainer
write a large-repository exclusion list from evidence rather than guesswork.

The run also produced the finding behind new task 1.4: 275 seconds for a
ten-line file. cost_usd is 0.0, so those 21 invocations were not model calls --
they were Joern query launches, one per region per family, each a JVM start.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-09 07:42:44 +02:00
co-authored by Claude Opus 5
parent 8663c1a5b3
commit e67f3ee2f0
7 changed files with 287 additions and 7 deletions
+5 -2
View File
@@ -41,8 +41,11 @@ def test_degradation_recorded_when_models_set_but_extra_absent(tmp_path: Path, m
degradations = manifest.get("degradations", [])
stages = {entry["stage"] for entry in degradations}
assert stages == {"hunter_pool", "verify"}
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in degradations)
# `model` is present because the model layer records that no CPG engine was available rather than being
# silently absent (contributor-scan Req 1.3). This test is about the HarnessX extra, so it asserts over
# the HarnessX stages specifically instead of the whole set.
assert stages == {"hunter_pool", "verify", "model"}
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in degradations if entry["stage"] != "model")
# Fallback still produced findings (deterministic hunter ran).
assert manifest["findings"]
+98
View File
@@ -0,0 +1,98 @@
"""contributor-scan task 1.3: the model layer reaches a user (Req 1).
`model-grounded-detection` finished 16/16 with the model layer having zero production callers: everything it
measured came from a harness. This is the stage that closes that, and it is deliberately additive — the
pattern, overlay and obligation findings all keep working, and a machine without a CPG engine scans exactly
as it does today.
"""
from __future__ import annotations
from pathlib import Path
import pytest
class _Runtime:
"""The two things `_run_model_layer` uses. A real HarnessRuntime needs a scan id, config and trace
writer, none of which this behaviour depends on."""
def __init__(self) -> None:
self.state: dict[str, list] = {"degradations": []}
def run_stage(self, name, fn): # type: ignore[no-untyped-def]
return fn()
def _repo(tmp_path: Path) -> Path:
(tmp_path / "app.py").write_text(
"import os\nfrom flask import request\n\n\ndef run():\n cmd = request.args['c']\n os.system(cmd)\n"
)
return tmp_path
def test_the_model_config_block_exists_and_is_conservative() -> None:
"""Req 8.4: optional planes default to something that cannot surprise a first-time user."""
from openultrasast.config import load_config
model = load_config(None).model
assert model.max_model_calls > 0 and model.max_regions > 0
assert model.max_model_calls <= 500, "a default budget should not be able to run away"
def test_the_config_is_readable_from_toml(tmp_path: Path) -> None:
from openultrasast.config import load_config
path = tmp_path / "ousast.toml"
path.write_text("[model]\nenabled = false\nmax_model_calls = 7\nmax_regions = 3\n")
model = load_config(path).model
assert model.enabled is False and model.max_model_calls == 7 and model.max_regions == 3
def test_without_a_cpg_engine_the_stage_degrades_and_records_why(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Req 1.3: the model layer is additive. A machine without Joern must scan exactly as it does today."""
from openultrasast.cli import _run_model_layer
from openultrasast.config import load_config
monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "off")
runtime = _Runtime()
findings, payload = _run_model_layer(
root=_repo(tmp_path), targets=[], entries=[], settings=load_config(None).model, runtime=runtime, config=load_config(None)
)
assert findings == []
assert any(d.get("reason") == "cpg_unavailable" for d in runtime.state["degradations"])
assert payload["skipped"] == "cpg_unavailable"
def test_the_payload_reports_what_the_model_arbitrated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Req 1.5: a user should see how much of a result the model established, not have to infer it."""
from openultrasast.cli import _run_model_layer
from openultrasast.config import load_config
monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "off")
_, payload = _run_model_layer(
root=_repo(tmp_path), targets=[], entries=[], settings=load_config(None).model, runtime=_Runtime(), config=load_config(None)
)
assert set(payload) >= {"by_rung", "regions_scanned", "regions_unjudged", "model_calls", "cost_usd"}
def test_unjudged_regions_are_reported_with_their_rank(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""What the budget could not reach is what a large-repository exclusion list should be built from, so the
lowest-ranked skipped regions are named rather than merely counted."""
from openultrasast.cli import _model_payload
from openultrasast.model.scan import ModelScanResult
result = ModelScanResult(by_rung={}, regions_scanned=2, regions_unjudged=8, model_calls=2, cost_usd=0.01)
payload = _model_payload(result, unjudged_paths=("low/a.py", "low/b.py"))
assert payload["regions_unjudged"] == 8
assert payload["unjudged_sample"] == ["low/a.py", "low/b.py"]
def test_a_disabled_model_layer_runs_nothing(tmp_path: Path) -> None:
from openultrasast.cli import _run_model_layer
from openultrasast.config import ModelLayerConfig, load_config
findings, payload = _run_model_layer(
root=_repo(tmp_path), targets=[], entries=[], settings=ModelLayerConfig(enabled=False), runtime=_Runtime(), config=load_config(None)
)
assert findings == [] and payload["skipped"] == "disabled"
+7 -3
View File
@@ -65,10 +65,14 @@ def test_standard_scan_extra_absent_matches_deterministic_baseline(tmp_path: Pat
assert requested == baseline # byte-identical findings -> sync driver behaviour unchanged
stages = {entry["stage"] for entry in requested_manifest.get("degradations", [])}
assert stages == {"hunter_pool", "verify"}
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in requested_manifest["degradations"])
# `model` joins the set when the model layer runs without a CPG engine: findings are byte-identical
# (asserted above) and the only difference is that the scan now SAYS the arbiter was unavailable
# (contributor-scan Req 1.3) rather than being silently absent.
assert stages == {"hunter_pool", "verify", "model"}
assert {e["reason"] for e in requested_manifest["degradations"] if e["stage"] == "model"} == {"cpg_unavailable"}
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in requested_manifest["degradations"] if entry["stage"] != "model")
assert all(entry["reason"] != "hunter_model_unavailable" for entry in requested_manifest["degradations"])
assert [entry["reason"] for entry in baseline_manifest.get("degradations", [])] == ["hunter_model_unavailable"]
assert [e["reason"] for e in baseline_manifest.get("degradations", []) if e["stage"] != "model"] == ["hunter_model_unavailable"]
def test_quick_scan_is_unaffected_by_the_extra(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: