mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
feat(harnessx): configurable provider + per-task budget caps
Add a [harnessx] config block so the agentic plane can be steered from
openultrasast.toml, and make the LLM provider selectable (the hunter and verifier
previously hardcoded AnthropicProvider).
- config.HarnessxConfig: provider ("anthropic"|"openai"|"litellm", default
anthropic), max_cost_usd (2.0), token_threshold (120000); parsed from [harnessx].
- harness_ext.build_provider: single lazy, capability-guarded provider factory.
The provider name is validated before any import, so an unknown choice fails with
a clear ValueError even when the extra is absent; each provider reads its own
standard key env var (openai -> OPENAI_API_KEY, anthropic -> ANTHROPIC_API_KEY).
- HxScanOrchestrator and verify_findings_dispatch take a provider and route through
build_provider; cli threads config.harnessx.{provider,max_cost_usd,token_threshold}
into both the hunter and the verifier.
- README: "Configuring the HarnessX agentic plane" section (install, models,
provider, budgets, fallback/degradation).
Test isolation: convert the "harnessx not in sys.modules" import-guards to a
hermetic subprocess fixture (tests/conftest.py assert_cold_of_harnessx) and
simulate extra-absent in the CLI dispatch tests, so the suite is deterministic
whether or not the optional extra is installed (it now is, locally). Verified the
OpenAI path constructs end-to-end (build_provider -> OpenAIProvider; orchestrator
-> Harness) with the budget threaded through.
137 passed, 3 skipped; ruff + mypy clean; detection gate PASS.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
760df7c809
commit
e255eb264f
@@ -169,7 +169,12 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
write_ranking_calibrations(calibrations, run.root / "calibration" / "applied_calibrations.json")
|
||||
if mode == "standard":
|
||||
if hx_hunter and hunter_model:
|
||||
orchestrator = HxScanOrchestrator(provider_model=hunter_model)
|
||||
orchestrator = HxScanOrchestrator(
|
||||
provider_model=hunter_model,
|
||||
provider=config.harnessx.provider,
|
||||
max_cost_usd=config.harnessx.max_cost_usd,
|
||||
token_threshold=config.harnessx.token_threshold,
|
||||
)
|
||||
hunter_result = runtime.run_stage(
|
||||
"hunter_pool",
|
||||
lambda: orchestrator.run_pool(
|
||||
@@ -222,7 +227,12 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
)
|
||||
verifications = runtime.run_stage(
|
||||
"verify",
|
||||
lambda: verify_findings_dispatch(findings, verifier_model=verifier_model, use_harnessx=hx_verify),
|
||||
lambda: verify_findings_dispatch(
|
||||
findings,
|
||||
verifier_model=verifier_model,
|
||||
verifier_provider=config.harnessx.provider,
|
||||
use_harnessx=hx_verify,
|
||||
),
|
||||
)
|
||||
write_verification_results(verifications, verification_path)
|
||||
runtime.run_stage(
|
||||
|
||||
@@ -66,6 +66,17 @@ class RulesetConfig:
|
||||
min_emit_precision: float = 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessxConfig:
|
||||
# Optional agentic plane (the openultrasast[harnessx] extra). `provider`
|
||||
# selects the HarnessX LLM provider used by both the hunter pool and the
|
||||
# llm-judge verifier; the budget caps bound per-task cost and token spend.
|
||||
# Inert unless the extra is installed and a model is configured in [models].
|
||||
provider: str = "anthropic"
|
||||
max_cost_usd: float = 2.0
|
||||
token_threshold: int = 120_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedConfig:
|
||||
models: ModelConfig = ModelConfig()
|
||||
@@ -76,6 +87,7 @@ class ResolvedConfig:
|
||||
static_analysis: StaticAnalysisConfig = StaticAnalysisConfig()
|
||||
score: ScoreConfig = ScoreConfig()
|
||||
ruleset: RulesetConfig = RulesetConfig()
|
||||
harnessx: HarnessxConfig = HarnessxConfig()
|
||||
runs_dir: str = ".openultrasast/runs"
|
||||
|
||||
|
||||
@@ -94,6 +106,7 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig:
|
||||
static_analysis=_load_static_analysis(data.get("static_analysis", {})),
|
||||
score=_load_score(data.get("score", {})),
|
||||
ruleset=_load_ruleset_config(data.get("ruleset", {})),
|
||||
harnessx=_load_harnessx(data.get("harnessx", {})),
|
||||
runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"),
|
||||
)
|
||||
|
||||
@@ -188,6 +201,15 @@ def _load_ruleset_config(value: object) -> RulesetConfig:
|
||||
)
|
||||
|
||||
|
||||
def _load_harnessx(value: object) -> HarnessxConfig:
|
||||
data = _section(value)
|
||||
return HarnessxConfig(
|
||||
provider=_string(data.get("provider")) or "anthropic",
|
||||
max_cost_usd=_float_value(data.get("max_cost_usd"), 2.0),
|
||||
token_threshold=_int_value(data.get("token_threshold"), 120_000),
|
||||
)
|
||||
|
||||
|
||||
def _float_value(value: object, default: float) -> float:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
|
||||
@@ -13,13 +13,17 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import importlib.util
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from harnessx.core.harness import HarnessConfig
|
||||
|
||||
HARNESSX_EXTRA = "openultrasast[harnessx]"
|
||||
|
||||
# HarnessX providers OpenUltraSAST can drive for the hunter / llm-judge stages.
|
||||
# Each reads its own standard API-key env var (ANTHROPIC_API_KEY / OPENAI_API_KEY).
|
||||
SUPPORTED_PROVIDERS = ("anthropic", "openai", "litellm")
|
||||
|
||||
|
||||
class HarnessXUnavailableError(RuntimeError):
|
||||
"""Raised when a HarnessX-backed capability is requested but the extra is absent."""
|
||||
@@ -37,6 +41,32 @@ def require_harnessx() -> ModuleType:
|
||||
return importlib.import_module("harnessx")
|
||||
|
||||
|
||||
def build_provider(model: str, provider: str = "anthropic") -> Any:
|
||||
"""Build a HarnessX LLM provider for ``model`` (lazy, capability-guarded).
|
||||
|
||||
The single place that knows HarnessX provider class names. The provider name is
|
||||
validated before any import so an unknown choice fails loudly even when the extra
|
||||
is absent; each provider reads its own standard API-key env var. Raises
|
||||
:class:`ValueError` on an unknown provider and :class:`HarnessXUnavailableError`
|
||||
when the extra is not installed.
|
||||
"""
|
||||
key = provider.lower()
|
||||
if key not in SUPPORTED_PROVIDERS:
|
||||
raise ValueError(f"unknown HarnessX provider {provider!r}; choose one of {SUPPORTED_PROVIDERS}")
|
||||
require_harnessx()
|
||||
if key == "anthropic":
|
||||
from harnessx.providers.anthropic_provider import AnthropicProvider
|
||||
|
||||
return AnthropicProvider(model=model)
|
||||
if key == "openai":
|
||||
from harnessx.providers.openai_provider import OpenAIProvider
|
||||
|
||||
return OpenAIProvider(model=model)
|
||||
from harnessx.providers.litellm_provider import LiteLLMProvider
|
||||
|
||||
return LiteLLMProvider(model=model)
|
||||
|
||||
|
||||
def build_sast(*, max_cost_usd: float = 2.0, token_threshold: int = 120_000) -> HarnessConfig:
|
||||
"""Compose the SAST hunter HarnessConfig (behaviour pipeline, model-free).
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .findings import SEVERITY_LABEL, StaticFinding, _reachability_conditions, _reachability_for_line
|
||||
from .harness_ext import build_sast, require_harnessx
|
||||
from .harness_ext import build_provider, build_sast, require_harnessx
|
||||
from .hunter import (
|
||||
HunterPoolResult,
|
||||
HunterTask,
|
||||
@@ -49,19 +49,26 @@ class ScanTask:
|
||||
class HxScanOrchestrator:
|
||||
"""Runs the hunter pool on a real HarnessX agent loop, one bounded task per target."""
|
||||
|
||||
def __init__(self, *, provider_model: str, max_cost_usd: float = 2.0, token_threshold: int = 120_000) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
provider_model: str,
|
||||
provider: str = "anthropic",
|
||||
max_cost_usd: float = 2.0,
|
||||
token_threshold: int = 120_000,
|
||||
) -> None:
|
||||
require_harnessx()
|
||||
self._provider_model = provider_model
|
||||
self._provider = provider
|
||||
self._max_cost_usd = max_cost_usd
|
||||
self._token_threshold = token_threshold
|
||||
self._agent: Any = None
|
||||
|
||||
def _build_agent(self) -> Any:
|
||||
from harnessx.core.model_config import ModelConfig
|
||||
from harnessx.providers.anthropic_provider import AnthropicProvider
|
||||
|
||||
config = build_sast(max_cost_usd=self._max_cost_usd, token_threshold=self._token_threshold)
|
||||
return ModelConfig(main=AnthropicProvider(model=self._provider_model)).agentic(config)
|
||||
return ModelConfig(main=build_provider(self._provider_model, self._provider)).agentic(config)
|
||||
|
||||
def run_pool(
|
||||
self,
|
||||
|
||||
@@ -8,7 +8,7 @@ so behaviour is byte-identical without HarnessX. The judge path is lazy/guarded.
|
||||
from __future__ import annotations
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import has_harnessx
|
||||
from .harness_ext import build_provider, has_harnessx
|
||||
from .verification import (
|
||||
EvidenceLevel,
|
||||
VerificationResult,
|
||||
@@ -23,6 +23,7 @@ def verify_findings_dispatch(
|
||||
findings: list[StaticFinding],
|
||||
*,
|
||||
verifier_model: str | None = None,
|
||||
verifier_provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
) -> list[VerificationResult]:
|
||||
"""Drop-in superset of :func:`verification.verify_findings`.
|
||||
@@ -32,21 +33,20 @@ def verify_findings_dispatch(
|
||||
"""
|
||||
if not (use_harnessx and verifier_model and has_harnessx()):
|
||||
return verify_findings(findings)
|
||||
return _verify_with_judge(findings, verifier_model)
|
||||
return _verify_with_judge(findings, verifier_model, verifier_provider)
|
||||
|
||||
|
||||
def _verify_with_judge(findings: list[StaticFinding], verifier_model: str) -> list[VerificationResult]:
|
||||
def _verify_with_judge(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
import asyncio
|
||||
|
||||
return asyncio.run(_judge_all(findings, verifier_model))
|
||||
return asyncio.run(_judge_all(findings, verifier_model, verifier_provider))
|
||||
|
||||
|
||||
async def _judge_all(findings: list[StaticFinding], verifier_model: str) -> list[VerificationResult]:
|
||||
async def _judge_all(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
from harnessx.providers.anthropic_provider import AnthropicProvider
|
||||
|
||||
provider = AnthropicProvider(model=verifier_model)
|
||||
provider = build_provider(verifier_model, verifier_provider)
|
||||
results: list[VerificationResult] = []
|
||||
for finding in findings:
|
||||
prompt = build_judge_prompt(
|
||||
|
||||
Reference in New Issue
Block a user