mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 4.8
ef59c0ae3d
Adopt HarnessX at the hunter loop and verifier, capability-gated so the default
zero-dependency scan is byte-identical and CI (no extra) exercises the fallback.
- 6.1/6.2 build_sast() (single, lazy HarnessX composition site in harness_ext.py,
mirrors DocuHarnessX make_docgen) + HxScanOrchestrator (hunter_harness.py): a
drop-in for run_hunter_pool that runs one bounded HarnessX agent task per ranked
target (ModelConfig(main=AnthropicProvider).agentic(build_sast()).run(BaseTask)),
parses findings from a JSON structured-output contract, returns the existing
HunterPoolResult, maps scan_id->session_id and emits per-target exit_reason/cost/
steps to the outer trace.
- 6.4 verify_judge.verify_findings_dispatch: a drop-in superset of
verification.verify_findings — structural verifier verbatim by default (rejects
unknown reachability), HarnessX llm-judge (build_judge_prompt/parse_judge_response)
when the extra + a verifier model are present; verdict mapped to VerificationResult
evidence-gated (plausible>=0.6 accepted; unsupported/refused rejected; infra
failure never auto-rejects).
- 6.5 cli capability gate (hx_hunter/hx_verify): selects HxScanOrchestrator /
judge only in standard mode with a configured model AND has_harnessx(); otherwise
falls back and records a {stage,reason:harnessx_extra_unavailable} degradation in
the manifest (new append-only write_manifest degradations kwarg). With no model
configured the manifest stays byte-identical.
- pyproject: mypy override ignore_missing_imports for harnessx.* (absent in the
type-check env).
Verified the HarnessX path constructs against the real extra in a throwaway venv
(build_sast -> HarnessConfig; orchestrator -> Harness; build_judge_prompt/
parse_judge_response/verdict mapping) — only the live agent.run()/provider.complete()
LLM calls are offline-untestable and stay gated. 109 tests pass (2 construction
smokes skip without the extra), ruff + mypy clean, detection gate 93.6%/0% intact.
Tasks 6.1/6.2/6.4/6.5 done; 6.3 (deterministic stages as HarnessX processors) deferred.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
62 lines
2.9 KiB
Python
62 lines
2.9 KiB
Python
"""Verifier dispatcher: structural fallback (default) + judge verdict mapping (Phase 3 task 6.4)."""
|
|
|
|
import sys
|
|
|
|
from openultrasast.findings import StaticFinding
|
|
from openultrasast.verification import VerificationStatus, verify_findings
|
|
from openultrasast.verify_judge import _verdict_to_verification, verify_findings_dispatch
|
|
|
|
|
|
def _finding(reachability: str = "reachable") -> StaticFinding:
|
|
return StaticFinding(
|
|
finding_id="python-unsafe-eval:app.py:3",
|
|
path="app.py",
|
|
title="Dynamic Python execution needs review",
|
|
severity="critical",
|
|
confidence="medium",
|
|
evidence_level="static_corroboration",
|
|
rationale="Static pattern matched.",
|
|
line=3,
|
|
function_name="admin" if reachability == "reachable" else None,
|
|
reachability_status=reachability,
|
|
reachability_evidence=[{"kind": "route", "access_level": "public", "line": 1, "end_line": 3}]
|
|
if reachability == "reachable"
|
|
else [],
|
|
reachability_conditions=[],
|
|
tags=["syscall_entry"],
|
|
ranking_priority=3.0,
|
|
)
|
|
|
|
|
|
def test_dispatch_falls_back_to_structural_by_default() -> None:
|
|
findings = [_finding("reachable"), _finding("unknown")]
|
|
assert verify_findings_dispatch(findings) == verify_findings(findings)
|
|
|
|
|
|
def test_dispatch_stays_structural_when_model_unset_or_extra_absent() -> None:
|
|
findings = [_finding("reachable")]
|
|
# use_harnessx requested but no model -> structural; and no model -> structural.
|
|
assert verify_findings_dispatch(findings, verifier_model=None, use_harnessx=True) == verify_findings(findings)
|
|
assert verify_findings_dispatch(findings, verifier_model="claude-x", use_harnessx=False) == verify_findings(findings)
|
|
|
|
|
|
def test_dispatch_does_not_import_harnessx_on_fallback() -> None:
|
|
verify_findings_dispatch([_finding()], verifier_model="claude-x", use_harnessx=True)
|
|
assert "harnessx" not in sys.modules
|
|
|
|
|
|
def test_verdict_mapping_is_evidence_gated() -> None:
|
|
finding = _finding("reachable")
|
|
accepted = _verdict_to_verification(finding, {"verdict": "plausible", "confidence": 0.9, "cause": "tainted sink reached"})
|
|
low_conf = _verdict_to_verification(finding, {"verdict": "plausible", "confidence": 0.3})
|
|
rejected = _verdict_to_verification(finding, {"verdict": "unsupported", "cause": "input is constant"})
|
|
hedging = _verdict_to_verification(finding, {"verdict": "hedging"})
|
|
infra = _verdict_to_verification(finding, {"verdict": "no_answer"})
|
|
|
|
assert accepted.status == VerificationStatus.ACCEPTED and accepted.verified
|
|
assert accepted.pro_case == "tainted sink reached"
|
|
assert low_conf.status == VerificationStatus.NEEDS_EVIDENCE
|
|
assert rejected.status == VerificationStatus.REJECTED and rejected.counter_case == "input is constant"
|
|
assert hedging.status == VerificationStatus.NEEDS_EVIDENCE
|
|
assert infra.status == VerificationStatus.NEEDS_EVIDENCE # infra failure never auto-rejects
|