mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 04:59:28 +00:00
feat(harnessx): Phase 3 task 6 — hunter sub-harness + llm-judge verifier + fallback
Adopt HarnessX at the hunter loop and verifier, capability-gated so the default
zero-dependency scan is byte-identical and CI (no extra) exercises the fallback.
- 6.1/6.2 build_sast() (single, lazy HarnessX composition site in harness_ext.py,
mirrors DocuHarnessX make_docgen) + HxScanOrchestrator (hunter_harness.py): a
drop-in for run_hunter_pool that runs one bounded HarnessX agent task per ranked
target (ModelConfig(main=AnthropicProvider).agentic(build_sast()).run(BaseTask)),
parses findings from a JSON structured-output contract, returns the existing
HunterPoolResult, maps scan_id->session_id and emits per-target exit_reason/cost/
steps to the outer trace.
- 6.4 verify_judge.verify_findings_dispatch: a drop-in superset of
verification.verify_findings — structural verifier verbatim by default (rejects
unknown reachability), HarnessX llm-judge (build_judge_prompt/parse_judge_response)
when the extra + a verifier model are present; verdict mapped to VerificationResult
evidence-gated (plausible>=0.6 accepted; unsupported/refused rejected; infra
failure never auto-rejects).
- 6.5 cli capability gate (hx_hunter/hx_verify): selects HxScanOrchestrator /
judge only in standard mode with a configured model AND has_harnessx(); otherwise
falls back and records a {stage,reason:harnessx_extra_unavailable} degradation in
the manifest (new append-only write_manifest degradations kwarg). With no model
configured the manifest stays byte-identical.
- pyproject: mypy override ignore_missing_imports for harnessx.* (absent in the
type-check env).
Verified the HarnessX path constructs against the real extra in a throwaway venv
(build_sast -> HarnessConfig; orchestrator -> Harness; build_judge_prompt/
parse_judge_response/verdict mapping) — only the live agent.run()/provider.complete()
LLM calls are offline-untestable and stay gated. 109 tests pass (2 construction
smokes skip without the extra), ruff + mypy clean, detection gate 93.6%/0% intact.
Tasks 6.1/6.2/6.4/6.5 done; 6.3 (deterministic stages as HarnessX processors) deferred.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
5e206df166
commit
ef59c0ae3d
@@ -0,0 +1,186 @@
|
||||
"""HarnessX-backed hunter pool — a real LLM agent loop per ranked target.
|
||||
|
||||
A drop-in for :func:`hunter.run_hunter_pool` (same call shape, returns the same
|
||||
:class:`hunter.HunterPoolResult`) that runs the optional HarnessX agentic plane. It
|
||||
is lazy/guarded: importing this module never imports ``harnessx``; the extra is only
|
||||
touched once :meth:`HxScanOrchestrator.run_pool` runs. The agent ``.run()`` path makes
|
||||
LLM API calls and is therefore exercised only with the extra + credentials present;
|
||||
the default scan keeps using the deterministic :func:`hunter.run_hunter_pool`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .findings import SEVERITY_LABEL, StaticFinding, _reachability_conditions, _reachability_for_line
|
||||
from .harness_ext import build_sast, require_harnessx
|
||||
from .hunter import (
|
||||
HunterPoolResult,
|
||||
HunterTask,
|
||||
HunterTrajectory,
|
||||
build_retrieval_context_by_path,
|
||||
schedule_hunter_tasks,
|
||||
)
|
||||
from .policy import CwePolicy, load_policy, resolve_severity
|
||||
from .preprocess import FileTarget
|
||||
from .rank import RankingScore
|
||||
from .ruleset import PatternRule
|
||||
|
||||
# The hunter agent is asked to emit findings as a JSON array of these objects.
|
||||
_FINDINGS_JSON = re.compile(r"\[\s*\{.*\}\s*\]", re.DOTALL)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScanTask:
|
||||
"""Thin description of one bounded hunter task (wraps, not subclasses, BaseTask)."""
|
||||
|
||||
description: str
|
||||
success_criteria: str
|
||||
max_steps: int
|
||||
token_budget: int | None
|
||||
max_cost_usd: float | None
|
||||
|
||||
|
||||
class HxScanOrchestrator:
|
||||
"""Runs the hunter pool on a real HarnessX agent loop, one bounded task per target."""
|
||||
|
||||
def __init__(self, *, provider_model: str, max_cost_usd: float = 2.0, token_threshold: int = 120_000) -> None:
|
||||
require_harnessx()
|
||||
self._provider_model = provider_model
|
||||
self._max_cost_usd = max_cost_usd
|
||||
self._token_threshold = token_threshold
|
||||
self._agent: Any = None
|
||||
|
||||
def _build_agent(self) -> Any:
|
||||
from harnessx.core.model_config import ModelConfig
|
||||
from harnessx.providers.anthropic_provider import AnthropicProvider
|
||||
|
||||
config = build_sast(max_cost_usd=self._max_cost_usd, token_threshold=self._token_threshold)
|
||||
return ModelConfig(main=AnthropicProvider(model=self._provider_model)).agentic(config)
|
||||
|
||||
def run_pool(
|
||||
self,
|
||||
root: Path,
|
||||
targets: list[FileTarget],
|
||||
rankings: list[RankingScore],
|
||||
*,
|
||||
scan_id: str,
|
||||
ruleset: tuple[PatternRule, ...] | None = None,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
emit: Callable[..., None] | None = None,
|
||||
) -> HunterPoolResult:
|
||||
import asyncio
|
||||
|
||||
tasks = schedule_hunter_tasks(targets, rankings, retrieval_context_by_path=build_retrieval_context_by_path(root, targets))
|
||||
resolved_policy = policy if policy is not None else load_policy()
|
||||
self._agent = self._agent or self._build_agent()
|
||||
return asyncio.run(self._run_all(scan_id, tasks, resolved_policy, emit))
|
||||
|
||||
async def _run_all(
|
||||
self,
|
||||
scan_id: str,
|
||||
tasks: list[HunterTask],
|
||||
policy: dict[str, CwePolicy],
|
||||
emit: Callable[..., None] | None,
|
||||
) -> HunterPoolResult:
|
||||
from harnessx.core.harness import BaseTask
|
||||
|
||||
findings: list[StaticFinding] = []
|
||||
trajectories: list[HunterTrajectory] = []
|
||||
agent = self._agent
|
||||
for task in tasks:
|
||||
base_task = BaseTask(
|
||||
description=self._describe(task),
|
||||
success_criteria="Enumerate each reachable tainted sink with file:line and a one-line rationale.",
|
||||
max_steps=task.budget.max_findings_per_target,
|
||||
token_budget=task.budget.retrieval_budget_chars * 4,
|
||||
max_cost_usd=self._max_cost_usd,
|
||||
)
|
||||
result = await agent.run(base_task, session_id=scan_id)
|
||||
task_findings = self._findings_from_result(task, result, policy)
|
||||
findings.extend(task_findings)
|
||||
trajectories.append(self._trajectory_from_result(scan_id, task, result, task_findings))
|
||||
if emit is not None:
|
||||
emit(
|
||||
"stage_end",
|
||||
f"hunter_pool:{task.target.path}",
|
||||
{
|
||||
"exit_reason": getattr(result, "exit_reason", "unknown"),
|
||||
"total_steps": getattr(result, "total_steps", 0),
|
||||
"total_cost_usd": getattr(result, "total_cost_usd", 0.0),
|
||||
},
|
||||
)
|
||||
await agent.cleanup()
|
||||
return HunterPoolResult(findings=sorted(findings, key=lambda item: item.finding_id), trajectories=trajectories)
|
||||
|
||||
def _describe(self, task: HunterTask) -> str:
|
||||
context = task.retrieval_context or f"[repo_code] {task.target.path}"
|
||||
return (
|
||||
f"Audit `{task.target.path}` ({task.target.language}) for security-relevant vulnerabilities. "
|
||||
f"Report ONLY evidence-backed findings as a JSON array, each object: "
|
||||
f'{{"line": <int>, "title": <str>, "cwe": "CWE-NN" (optional), "rationale": <str>}}. '
|
||||
f"Emit `[]` if none.\n\nSource context:\n{context}"
|
||||
)
|
||||
|
||||
def _findings_from_result(self, task: HunterTask, result: object, policy: dict[str, CwePolicy]) -> list[StaticFinding]:
|
||||
items = _extract_findings(getattr(result, "final_output", "") or "")
|
||||
findings: list[StaticFinding] = []
|
||||
for item in items:
|
||||
line = item.get("line")
|
||||
line = int(line) if isinstance(line, int) else None
|
||||
reachability_status, function_name, reachability_evidence = (
|
||||
_reachability_for_line(task.target, line) if line is not None else ("unknown", None, [])
|
||||
)
|
||||
cwe = str(item.get("cwe", "")) or None
|
||||
severity = SEVERITY_LABEL.get(resolve_severity(policy, cwe), "medium") if cwe else "medium"
|
||||
line_part = line if line is not None else 0
|
||||
findings.append(
|
||||
StaticFinding(
|
||||
finding_id=f"hx-hunter:{task.target.path}:{line_part}",
|
||||
path=task.target.path,
|
||||
title=str(item.get("title", "HarnessX hunter finding")),
|
||||
severity=severity,
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale=str(item.get("rationale", "Reported by the HarnessX hunter; manual verification required.")),
|
||||
line=line,
|
||||
function_name=function_name,
|
||||
reachability_status=reachability_status,
|
||||
reachability_evidence=reachability_evidence,
|
||||
reachability_conditions=_reachability_conditions(reachability_evidence),
|
||||
tags=sorted(set(task.target.tags)),
|
||||
ranking_priority=task.ranking.priority,
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
def _trajectory_from_result(
|
||||
self, scan_id: str, task: HunterTask, result: object, task_findings: list[StaticFinding]
|
||||
) -> HunterTrajectory:
|
||||
return HunterTrajectory(
|
||||
scan_id=scan_id,
|
||||
target_path=task.target.path,
|
||||
tier=task.tier,
|
||||
priority=task.ranking.priority,
|
||||
selected_skills=task.selected_skills,
|
||||
retrieval_context_chars=len(task.retrieval_context),
|
||||
findings=[finding.finding_id for finding in task_findings],
|
||||
status=str(getattr(result, "exit_reason", "completed")),
|
||||
)
|
||||
|
||||
|
||||
def _extract_findings(text: str) -> list[dict[str, object]]:
|
||||
"""Best-effort extraction of the findings JSON array from the agent's final output."""
|
||||
match = _FINDINGS_JSON.search(text)
|
||||
if match is None:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [item for item in parsed if isinstance(item, dict)] if isinstance(parsed, list) else []
|
||||
Reference in New Issue
Block a user