mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 4.8
ef59c0ae3d
Adopt HarnessX at the hunter loop and verifier, capability-gated so the default
zero-dependency scan is byte-identical and CI (no extra) exercises the fallback.
- 6.1/6.2 build_sast() (single, lazy HarnessX composition site in harness_ext.py,
mirrors DocuHarnessX make_docgen) + HxScanOrchestrator (hunter_harness.py): a
drop-in for run_hunter_pool that runs one bounded HarnessX agent task per ranked
target (ModelConfig(main=AnthropicProvider).agentic(build_sast()).run(BaseTask)),
parses findings from a JSON structured-output contract, returns the existing
HunterPoolResult, maps scan_id->session_id and emits per-target exit_reason/cost/
steps to the outer trace.
- 6.4 verify_judge.verify_findings_dispatch: a drop-in superset of
verification.verify_findings — structural verifier verbatim by default (rejects
unknown reachability), HarnessX llm-judge (build_judge_prompt/parse_judge_response)
when the extra + a verifier model are present; verdict mapped to VerificationResult
evidence-gated (plausible>=0.6 accepted; unsupported/refused rejected; infra
failure never auto-rejects).
- 6.5 cli capability gate (hx_hunter/hx_verify): selects HxScanOrchestrator /
judge only in standard mode with a configured model AND has_harnessx(); otherwise
falls back and records a {stage,reason:harnessx_extra_unavailable} degradation in
the manifest (new append-only write_manifest degradations kwarg). With no model
configured the manifest stays byte-identical.
- pyproject: mypy override ignore_missing_imports for harnessx.* (absent in the
type-check env).
Verified the HarnessX path constructs against the real extra in a throwaway venv
(build_sast -> HarnessConfig; orchestrator -> Harness; build_judge_prompt/
parse_judge_response/verdict mapping) — only the live agent.run()/provider.complete()
LLM calls are offline-untestable and stay gated. 109 tests pass (2 construction
smokes skip without the extra), ruff + mypy clean, detection gate 93.6%/0% intact.
Tasks 6.1/6.2/6.4/6.5 done; 6.3 (deterministic stages as HarnessX processors) deferred.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
57 lines
2.3 KiB
Python
57 lines
2.3 KiB
Python
"""CLI capability-gated dispatch + graceful degradation (Phase 3 task 6.5).
|
|
|
|
These run in the default (extra-absent) environment: standard mode with models
|
|
configured must fall back to the deterministic hunter/structural verifier and
|
|
record a degradation; with no models configured the manifest stays byte-identical.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openultrasast import cli
|
|
from openultrasast.cli import main
|
|
|
|
_VULN = "@app.route('/admin')\ndef admin():\n return eval(request.data)\n"
|
|
|
|
|
|
def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_body: str | None) -> dict:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text(_VULN)
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
argv = ["scan", str(repo), "--mode", "standard"]
|
|
if config_body is not None:
|
|
cfg = tmp_path / "openultrasast.toml"
|
|
cfg.write_text(config_body)
|
|
argv += ["--config", str(cfg)]
|
|
assert main(argv) == 0
|
|
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
|
return json.loads((run_dir / "manifest.json").read_text())
|
|
|
|
|
|
def test_degradation_recorded_when_models_set_but_extra_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
assert not cli.has_harnessx() # this CI env has no harnessx extra
|
|
manifest = _run(tmp_path, monkeypatch, '[models]\nhunter = "anthropic/claude"\nverifier = "anthropic/claude"\n')
|
|
|
|
degradations = manifest.get("degradations", [])
|
|
stages = {entry["stage"] for entry in degradations}
|
|
assert stages == {"hunter_pool", "verify"}
|
|
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in degradations)
|
|
# Fallback still produced findings (deterministic hunter ran).
|
|
assert manifest["findings"]
|
|
|
|
|
|
def test_manifest_has_no_degradations_without_models(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
manifest = _run(tmp_path, monkeypatch, config_body=None)
|
|
assert "degradations" not in manifest
|
|
|
|
|
|
def test_standard_scan_falls_back_to_run_hunter_pool(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls: list[str] = []
|
|
real = cli.run_hunter_pool
|
|
monkeypatch.setattr(cli, "run_hunter_pool", lambda *a, **k: calls.append("fallback") or real(*a, **k))
|
|
_run(tmp_path, monkeypatch, '[models]\nhunter = "anthropic/claude"\n')
|
|
assert calls == ["fallback"]
|