mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
e67f3ee2f0
`model-grounded-detection` finished 16/16 with the model layer having zero
production callers. This closes that. A real scan of a Flask file now reports:
by_rung: {"model_entailed": 3, "model_corroborated": 0, "suspicion": 0}
model:injection:ping:10 <- os.system("ping -c 1 " + request.args["host"])
The stage is the fifth MAP proposer, mirroring _check_obligations, and is
additive: pattern, overlay and obligation findings all keep working, and without
a CPG engine it records cpg_unavailable and changes nothing else. Two scan
integrity tests pin the exact degradation set and both broke -- correctly. Their
`requested == baseline` assertion, that findings are byte-identical, still
passed, so the only change is that the scan now SAYS the arbiter was absent
instead of being silently without one. The guards are updated to assert the
model degradation specifically rather than being loosened.
ModelLayerConfig, not ModelConfig: that name is already the [models] block, and
the type checker caught the collision immediately. Two meanings of one name is
how a scan silently reads the wrong settings.
Every payload carries the same shape, skipped ones included. A payload that is
sometimes a full record and sometimes a single "skipped" key is how a report
ends up quietly omitting a section.
unjudged_sample names the lowest-ranked regions the budget did not reach.
Counting them is the honesty requirement; naming them is what lets a maintainer
write a large-repository exclusion list from evidence rather than guesswork.
The run also produced the finding behind new task 1.4: 275 seconds for a
ten-line file. cost_usd is 0.0, so those 21 invocations were not model calls --
they were Joern query launches, one per region per family, each a JVM start.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
66 lines
3.1 KiB
Python
66 lines
3.1 KiB
Python
"""CLI capability-gated dispatch + graceful degradation (Phase 3 task 6.5).
|
|
|
|
Simulates the extra-absent environment (via a monkeypatched capability probe so the
|
|
behaviour is deterministic whether or not the optional extra is installed): standard
|
|
mode with models configured must fall back to the deterministic hunter/structural
|
|
verifier and record a degradation; with no models configured the manifest stays
|
|
byte-identical.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openultrasast import cli
|
|
from openultrasast.cli import main
|
|
|
|
_VULN = "@app.route('/admin')\ndef admin():\n return eval(request.data)\n"
|
|
|
|
|
|
def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_body: str | None) -> dict:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text(_VULN)
|
|
monkeypatch.setattr(cli, "has_harnessx", lambda: False) # simulate the extra being absent
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False)
|
|
argv = ["scan", str(repo), "--mode", "standard"]
|
|
if config_body is not None:
|
|
cfg = tmp_path / "openultrasast.toml"
|
|
cfg.write_text(config_body)
|
|
argv += ["--config", str(cfg)]
|
|
assert main(argv) == 0
|
|
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
|
return json.loads((run_dir / "manifest.json").read_text())
|
|
|
|
|
|
def test_degradation_recorded_when_models_set_but_extra_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
manifest = _run(tmp_path, monkeypatch, '[models]\nhunter = "anthropic/claude"\nverifier = "anthropic/claude"\n')
|
|
|
|
degradations = manifest.get("degradations", [])
|
|
stages = {entry["stage"] for entry in degradations}
|
|
# `model` is present because the model layer records that no CPG engine was available rather than being
|
|
# silently absent (contributor-scan Req 1.3). This test is about the HarnessX extra, so it asserts over
|
|
# the HarnessX stages specifically instead of the whole set.
|
|
assert stages == {"hunter_pool", "verify", "model"}
|
|
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in degradations if entry["stage"] != "model")
|
|
# Fallback still produced findings (deterministic hunter ran).
|
|
assert manifest["findings"]
|
|
|
|
|
|
def test_manifest_has_no_harnessx_degradations_without_models(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
manifest = _run(tmp_path, monkeypatch, config_body=None)
|
|
degradations = manifest.get("degradations", [])
|
|
assert all(entry.get("reason") != "harnessx_extra_unavailable" for entry in degradations)
|
|
assert any(entry.get("reason") == "hunter_model_unavailable" for entry in degradations)
|
|
|
|
|
|
def test_standard_scan_falls_back_to_run_hunter_pool(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls: list[str] = []
|
|
real = cli.run_hunter_pool
|
|
monkeypatch.setattr(cli, "run_hunter_pool", lambda *a, **k: calls.append("fallback") or real(*a, **k))
|
|
_run(tmp_path, monkeypatch, '[models]\nhunter = "anthropic/claude"\n')
|
|
assert calls == ["fallback"]
|