mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
feat(harnessx): configurable provider + per-task budget caps
Add a [harnessx] config block so the agentic plane can be steered from
openultrasast.toml, and make the LLM provider selectable (the hunter and verifier
previously hardcoded AnthropicProvider).
- config.HarnessxConfig: provider ("anthropic"|"openai"|"litellm", default
anthropic), max_cost_usd (2.0), token_threshold (120000); parsed from [harnessx].
- harness_ext.build_provider: single lazy, capability-guarded provider factory.
The provider name is validated before any import, so an unknown choice fails with
a clear ValueError even when the extra is absent; each provider reads its own
standard key env var (openai -> OPENAI_API_KEY, anthropic -> ANTHROPIC_API_KEY).
- HxScanOrchestrator and verify_findings_dispatch take a provider and route through
build_provider; cli threads config.harnessx.{provider,max_cost_usd,token_threshold}
into both the hunter and the verifier.
- README: "Configuring the HarnessX agentic plane" section (install, models,
provider, budgets, fallback/degradation).
Test isolation: convert the "harnessx not in sys.modules" import-guards to a
hermetic subprocess fixture (tests/conftest.py assert_cold_of_harnessx) and
simulate extra-absent in the CLI dispatch tests, so the suite is deterministic
whether or not the optional extra is installed (it now is, locally). Verified the
OpenAI path constructs end-to-end (build_provider -> OpenAIProvider; orchestrator
-> Harness) with the budget threaded through.
137 passed, 3 skipped; ruff + mypy clean; detection gate PASS.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
760df7c809
commit
e255eb264f
@@ -8,7 +8,7 @@ so behaviour is byte-identical without HarnessX. The judge path is lazy/guarded.
|
||||
from __future__ import annotations
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import has_harnessx
|
||||
from .harness_ext import build_provider, has_harnessx
|
||||
from .verification import (
|
||||
EvidenceLevel,
|
||||
VerificationResult,
|
||||
@@ -23,6 +23,7 @@ def verify_findings_dispatch(
|
||||
findings: list[StaticFinding],
|
||||
*,
|
||||
verifier_model: str | None = None,
|
||||
verifier_provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
) -> list[VerificationResult]:
|
||||
"""Drop-in superset of :func:`verification.verify_findings`.
|
||||
@@ -32,21 +33,20 @@ def verify_findings_dispatch(
|
||||
"""
|
||||
if not (use_harnessx and verifier_model and has_harnessx()):
|
||||
return verify_findings(findings)
|
||||
return _verify_with_judge(findings, verifier_model)
|
||||
return _verify_with_judge(findings, verifier_model, verifier_provider)
|
||||
|
||||
|
||||
def _verify_with_judge(findings: list[StaticFinding], verifier_model: str) -> list[VerificationResult]:
|
||||
def _verify_with_judge(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
import asyncio
|
||||
|
||||
return asyncio.run(_judge_all(findings, verifier_model))
|
||||
return asyncio.run(_judge_all(findings, verifier_model, verifier_provider))
|
||||
|
||||
|
||||
async def _judge_all(findings: list[StaticFinding], verifier_model: str) -> list[VerificationResult]:
|
||||
async def _judge_all(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
from harnessx.providers.anthropic_provider import AnthropicProvider
|
||||
|
||||
provider = AnthropicProvider(model=verifier_model)
|
||||
provider = build_provider(verifier_model, verifier_provider)
|
||||
results: list[VerificationResult] = []
|
||||
for finding in findings:
|
||||
prompt = build_judge_prompt(
|
||||
|
||||
Reference in New Issue
Block a user