feat(harnessx): configurable provider + per-task budget caps

Add a [harnessx] config block so the agentic plane can be steered from
openultrasast.toml, and make the LLM provider selectable (the hunter and verifier
previously hardcoded AnthropicProvider).

- config.HarnessxConfig: provider ("anthropic"|"openai"|"litellm", default
  anthropic), max_cost_usd (2.0), token_threshold (120000); parsed from [harnessx].
- harness_ext.build_provider: single lazy, capability-guarded provider factory.
  The provider name is validated before any import, so an unknown choice fails with
  a clear ValueError even when the extra is absent; each provider reads its own
  standard key env var (openai -> OPENAI_API_KEY, anthropic -> ANTHROPIC_API_KEY).
- HxScanOrchestrator and verify_findings_dispatch take a provider and route through
  build_provider; cli threads config.harnessx.{provider,max_cost_usd,token_threshold}
  into both the hunter and the verifier.
- README: "Configuring the HarnessX agentic plane" section (install, models,
  provider, budgets, fallback/degradation).

Test isolation: convert the "harnessx not in sys.modules" import-guards to a
hermetic subprocess fixture (tests/conftest.py assert_cold_of_harnessx) and
simulate extra-absent in the CLI dispatch tests, so the suite is deterministic
whether or not the optional extra is installed (it now is, locally). Verified the
OpenAI path constructs end-to-end (build_provider -> OpenAIProvider; orchestrator
-> Harness) with the budget threaded through.

137 passed, 3 skipped; ruff + mypy clean; detection gate PASS.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-06-22 17:59:44 +02:00
co-authored by Claude Opus 4.8
parent 760df7c809
commit e255eb264f
12 changed files with 240 additions and 40 deletions
+38 -11
View File
@@ -1,13 +1,19 @@
"""HarnessX optional-extra packaging + capability guard (Phase 3 task 5.2)."""
import re
import sys
import tomllib
from pathlib import Path
import pytest
from openultrasast.harness_ext import HARNESSX_EXTRA, HarnessXUnavailableError, has_harnessx, require_harnessx
from openultrasast.harness_ext import (
HARNESSX_EXTRA,
SUPPORTED_PROVIDERS,
HarnessXUnavailableError,
build_provider,
has_harnessx,
require_harnessx,
)
PYPROJECT = tomllib.loads(Path("pyproject.toml").read_text())
@@ -25,18 +31,15 @@ def test_harnessx_is_a_sha_pinned_optional_extra() -> None:
), extras[0]
def test_importing_openultrasast_does_not_import_harnessx() -> None:
def test_importing_openultrasast_does_not_import_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
# The big modules (cli, harness_ext) must not eagerly import the heavy extra.
import openultrasast.cli # noqa: F401
import openultrasast.harness_ext # noqa: F401
assert "harnessx" not in sys.modules
assert_cold_of_harnessx("import openultrasast.cli\nimport openultrasast.harness_ext")
def test_has_harnessx_probes_without_importing() -> None:
available = has_harnessx()
assert isinstance(available, bool)
assert "harnessx" not in sys.modules # find_spec must not import the module
def test_has_harnessx_probes_without_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
assert isinstance(has_harnessx(), bool)
# find_spec must not import the module.
assert_cold_of_harnessx("from openultrasast.harness_ext import has_harnessx\nassert isinstance(has_harnessx(), bool)")
def test_require_harnessx_raises_with_install_guidance_when_absent() -> None:
@@ -44,3 +47,27 @@ def test_require_harnessx_raises_with_install_guidance_when_absent() -> None:
pytest.skip("HarnessX extra is installed in this environment")
with pytest.raises(HarnessXUnavailableError, match=re.escape(HARNESSX_EXTRA)):
require_harnessx()
def test_build_provider_rejects_unknown_provider_before_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
# Name is validated before any import, so this fails loudly even without the extra.
with pytest.raises(ValueError, match="unknown HarnessX provider"):
build_provider("gpt-4o", "bogus")
assert_cold_of_harnessx(
"from openultrasast.harness_ext import build_provider\n"
"try:\n build_provider('gpt-4o', 'bogus')\n"
"except ValueError:\n pass\n"
"else:\n raise SystemExit('expected ValueError')\n"
)
def test_supported_providers_cover_anthropic_and_openai() -> None:
assert {"anthropic", "openai"} <= set(SUPPORTED_PROVIDERS)
@pytest.mark.parametrize(("provider", "class_name"), [("anthropic", "AnthropicProvider"), ("openai", "OpenAIProvider")])
def test_build_provider_constructs_selected_provider(provider: str, class_name: str) -> None:
if not has_harnessx():
pytest.skip("HarnessX extra not installed in this environment")
built = build_provider("test-model", provider)
assert type(built).__name__ == class_name