mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(harnessx): configurable provider + per-task budget caps
Add a [harnessx] config block so the agentic plane can be steered from
openultrasast.toml, and make the LLM provider selectable (the hunter and verifier
previously hardcoded AnthropicProvider).
- config.HarnessxConfig: provider ("anthropic"|"openai"|"litellm", default
anthropic), max_cost_usd (2.0), token_threshold (120000); parsed from [harnessx].
- harness_ext.build_provider: single lazy, capability-guarded provider factory.
The provider name is validated before any import, so an unknown choice fails with
a clear ValueError even when the extra is absent; each provider reads its own
standard key env var (openai -> OPENAI_API_KEY, anthropic -> ANTHROPIC_API_KEY).
- HxScanOrchestrator and verify_findings_dispatch take a provider and route through
build_provider; cli threads config.harnessx.{provider,max_cost_usd,token_threshold}
into both the hunter and the verifier.
- README: "Configuring the HarnessX agentic plane" section (install, models,
provider, budgets, fallback/degradation).
Test isolation: convert the "harnessx not in sys.modules" import-guards to a
hermetic subprocess fixture (tests/conftest.py assert_cold_of_harnessx) and
simulate extra-absent in the CLI dispatch tests, so the suite is deterministic
whether or not the optional extra is installed (it now is, locally). Verified the
OpenAI path constructs end-to-end (build_provider -> OpenAIProvider; orchestrator
-> Harness) with the budget threaded through.
137 passed, 3 skipped; ruff + mypy clean; detection gate PASS.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
760df7c809
commit
e255eb264f
+38
-11
@@ -1,13 +1,19 @@
|
||||
"""HarnessX optional-extra packaging + capability guard (Phase 3 task 5.2)."""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.harness_ext import HARNESSX_EXTRA, HarnessXUnavailableError, has_harnessx, require_harnessx
|
||||
from openultrasast.harness_ext import (
|
||||
HARNESSX_EXTRA,
|
||||
SUPPORTED_PROVIDERS,
|
||||
HarnessXUnavailableError,
|
||||
build_provider,
|
||||
has_harnessx,
|
||||
require_harnessx,
|
||||
)
|
||||
|
||||
PYPROJECT = tomllib.loads(Path("pyproject.toml").read_text())
|
||||
|
||||
@@ -25,18 +31,15 @@ def test_harnessx_is_a_sha_pinned_optional_extra() -> None:
|
||||
), extras[0]
|
||||
|
||||
|
||||
def test_importing_openultrasast_does_not_import_harnessx() -> None:
|
||||
def test_importing_openultrasast_does_not_import_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# The big modules (cli, harness_ext) must not eagerly import the heavy extra.
|
||||
import openultrasast.cli # noqa: F401
|
||||
import openultrasast.harness_ext # noqa: F401
|
||||
|
||||
assert "harnessx" not in sys.modules
|
||||
assert_cold_of_harnessx("import openultrasast.cli\nimport openultrasast.harness_ext")
|
||||
|
||||
|
||||
def test_has_harnessx_probes_without_importing() -> None:
|
||||
available = has_harnessx()
|
||||
assert isinstance(available, bool)
|
||||
assert "harnessx" not in sys.modules # find_spec must not import the module
|
||||
def test_has_harnessx_probes_without_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
assert isinstance(has_harnessx(), bool)
|
||||
# find_spec must not import the module.
|
||||
assert_cold_of_harnessx("from openultrasast.harness_ext import has_harnessx\nassert isinstance(has_harnessx(), bool)")
|
||||
|
||||
|
||||
def test_require_harnessx_raises_with_install_guidance_when_absent() -> None:
|
||||
@@ -44,3 +47,27 @@ def test_require_harnessx_raises_with_install_guidance_when_absent() -> None:
|
||||
pytest.skip("HarnessX extra is installed in this environment")
|
||||
with pytest.raises(HarnessXUnavailableError, match=re.escape(HARNESSX_EXTRA)):
|
||||
require_harnessx()
|
||||
|
||||
|
||||
def test_build_provider_rejects_unknown_provider_before_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# Name is validated before any import, so this fails loudly even without the extra.
|
||||
with pytest.raises(ValueError, match="unknown HarnessX provider"):
|
||||
build_provider("gpt-4o", "bogus")
|
||||
assert_cold_of_harnessx(
|
||||
"from openultrasast.harness_ext import build_provider\n"
|
||||
"try:\n build_provider('gpt-4o', 'bogus')\n"
|
||||
"except ValueError:\n pass\n"
|
||||
"else:\n raise SystemExit('expected ValueError')\n"
|
||||
)
|
||||
|
||||
|
||||
def test_supported_providers_cover_anthropic_and_openai() -> None:
|
||||
assert {"anthropic", "openai"} <= set(SUPPORTED_PROVIDERS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("provider", "class_name"), [("anthropic", "AnthropicProvider"), ("openai", "OpenAIProvider")])
|
||||
def test_build_provider_constructs_selected_provider(provider: str, class_name: str) -> None:
|
||||
if not has_harnessx():
|
||||
pytest.skip("HarnessX extra not installed in this environment")
|
||||
built = build_provider("test-model", provider)
|
||||
assert type(built).__name__ == class_name
|
||||
|
||||
Reference in New Issue
Block a user