mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
refactor(harnessx-removal): retire the HarnessX scan paths (task 5)
The standard scan now runs run_hunter_pool, verify_findings and the deterministic fuse_findings directly; the HarnessX hunter orchestrator, the LLM judge dispatcher (verify_judge.py, deleted with its test) and the LLM fusion panels are gone, and so are their extra-absent degradations. host_under_harnessx is dropped from stage_processors. config: [models] verifier and [fusion] panel_model/decider_model raise RetiredConfigError naming the plane replacement (the CLI prints it and exits 2); a [harnessx] section loads with one warning and is otherwise ignored. harness.json keeps "verifier": null. Tests that asserted the fallback are renamed to the behaviour they assert; the module-audit manifest drops verify_judge. Nothing outside the two modules deleted in task 6 imports harness_ext or hunter_harness. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
b68fe27c74
commit
cbadc38d5e
+9
-137
@@ -7,11 +7,9 @@ disclosure, or an explicit high-assurance request. Two panels independently revi
|
||||
bounded evidence (one steel-manning the vulnerability case, one the false-positive
|
||||
case), critique and revise, vote, and a decider issues the final disposition.
|
||||
|
||||
The deterministic engine here is the zero-dependency core (and the fallback). When the
|
||||
``openultrasast[harnessx]`` extra and panel models are configured, ``fuse_findings_dispatch``
|
||||
routes the panels through real LLM reviews; the decider reconciliation stays deterministic
|
||||
and auditable. Every fused finding receives exactly one of the five dispositions, and the
|
||||
decision discloses model IDs, panel roles, votes, degradations, and the decision source.
|
||||
The panels and the decider are deterministic and auditable. Every fused finding receives
|
||||
exactly one of the five dispositions, and the decision discloses panel roles, votes and the
|
||||
decision source. Independent LLM agreement is the plane's ``agree`` task, not this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,7 +19,6 @@ from dataclasses import asdict, dataclass, field
|
||||
from enum import StrEnum
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import has_harnessx
|
||||
from .verification import VerificationResult, VerificationStatus
|
||||
|
||||
|
||||
@@ -57,7 +54,7 @@ class FusionDecision:
|
||||
triggered: bool
|
||||
triggers: list[str]
|
||||
disposition: FusionDisposition
|
||||
decision_source: str # "deterministic-reconciler" | "llm-panels" | "no-fusion"
|
||||
decision_source: str # "deterministic-reconciler" | "no-fusion"
|
||||
panels: list[PanelVerdict]
|
||||
votes: dict[str, int]
|
||||
model_ids: list[str]
|
||||
@@ -191,155 +188,30 @@ def fuse_finding(
|
||||
)
|
||||
|
||||
|
||||
def fuse_findings_dispatch(
|
||||
def fuse_findings(
|
||||
findings: list[StaticFinding],
|
||||
verifications: list[VerificationResult],
|
||||
*,
|
||||
panel_model: str | None = None,
|
||||
decider_model: str | None = None,
|
||||
provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
high_assurance: bool = False,
|
||||
mitigated_ids: frozenset[str] = frozenset(),
|
||||
) -> list[FusionDecision]:
|
||||
"""Fuse every finding that the trigger policy selects.
|
||||
|
||||
Deterministic by default. When ``use_harnessx`` is set, a panel model is configured,
|
||||
and the extra is present, the panels are produced by real LLM reviews; otherwise the
|
||||
deterministic panels run and (if an LLM run was requested) a degradation is recorded.
|
||||
"""
|
||||
"""Fuse every finding that the trigger policy selects, with the deterministic panels."""
|
||||
verification_by_id = {result.finding_id: result for result in verifications}
|
||||
want_llm = use_harnessx and bool(panel_model)
|
||||
decisions: list[FusionDecision] = []
|
||||
for finding in findings:
|
||||
verification = verification_by_id.get(finding.finding_id)
|
||||
triggers = fusion_triggers(finding, verification, high_assurance=high_assurance)
|
||||
if not triggers:
|
||||
continue
|
||||
if want_llm and has_harnessx():
|
||||
decisions.append(_fuse_with_llm_panels(finding, verification, triggers, panel_model, decider_model, provider, mitigated_ids))
|
||||
else:
|
||||
decision = fuse_finding(finding, verification, triggers, mitigated_ids=mitigated_ids)
|
||||
if want_llm and not has_harnessx():
|
||||
decision = _with_degradation(decision, "fusion_llm_unavailable: harnessx extra absent; used deterministic panels")
|
||||
decisions.append(decision)
|
||||
if triggers:
|
||||
decisions.append(fuse_finding(finding, verification, triggers, mitigated_ids=mitigated_ids))
|
||||
return decisions
|
||||
|
||||
|
||||
def _with_degradation(decision: FusionDecision, note: str) -> FusionDecision:
|
||||
return FusionDecision(
|
||||
finding_id=decision.finding_id,
|
||||
triggered=decision.triggered,
|
||||
triggers=decision.triggers,
|
||||
disposition=decision.disposition,
|
||||
decision_source=decision.decision_source,
|
||||
panels=decision.panels,
|
||||
votes=decision.votes,
|
||||
model_ids=decision.model_ids,
|
||||
degradations=[*decision.degradations, note],
|
||||
warnings=decision.warnings,
|
||||
)
|
||||
|
||||
|
||||
def _fuse_with_llm_panels(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
triggers: list[str],
|
||||
panel_model: str | None,
|
||||
decider_model: str | None,
|
||||
provider: str,
|
||||
mitigated_ids: frozenset[str],
|
||||
) -> FusionDecision:
|
||||
"""LLM-backed panels with a deterministic, auditable decider (lazy/guarded)."""
|
||||
import asyncio
|
||||
|
||||
model = panel_model or ""
|
||||
try:
|
||||
panels = asyncio.run(_run_llm_panels(finding, verification, model, provider))
|
||||
source = "llm-panels"
|
||||
warnings: list[str] = []
|
||||
except Exception as exc: # noqa: BLE001 — panel/provider failure falls back, never crashes the scan
|
||||
panels = [_panel_a(finding, verification), _panel_b(finding, verification)]
|
||||
source = "deterministic-reconciler"
|
||||
warnings = [f"llm_panels_failed: {type(exc).__name__}"]
|
||||
disposition, _rationale = _reconcile(finding, verification, panels, mitigated_ids)
|
||||
votes = Counter(str(panel.disposition) for panel in panels)
|
||||
model_ids = sorted({panel.model_id for panel in panels if panel.model_id})
|
||||
if decider_model:
|
||||
model_ids = sorted({*model_ids, decider_model})
|
||||
return FusionDecision(
|
||||
finding_id=finding.finding_id,
|
||||
triggered=True,
|
||||
triggers=triggers,
|
||||
disposition=disposition,
|
||||
decision_source=source,
|
||||
panels=panels,
|
||||
votes=dict(sorted(votes.items())),
|
||||
model_ids=model_ids,
|
||||
degradations=[],
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def _run_llm_panels(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
model: str,
|
||||
provider: str,
|
||||
) -> list[PanelVerdict]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
|
||||
from .harness_ext import build_provider
|
||||
|
||||
client = build_provider(model, provider)
|
||||
panels: list[PanelVerdict] = []
|
||||
for role, leaning, stance in (
|
||||
("panel_a", "vulnerability", "Argue the vulnerability case as strongly as the evidence allows."),
|
||||
("panel_b", "false_positive", "Argue the false-positive case as strongly as the evidence allows."),
|
||||
):
|
||||
summary = f"reachability={finding.reachability_status}; severity={finding.severity}; tags={', '.join(finding.tags) or 'none'}"
|
||||
prompt = build_judge_prompt(
|
||||
task_description=f"{stance} Finding: {finding.title} at {finding.path}:{finding.line}.",
|
||||
trajectory_summary=summary,
|
||||
extracted_answer=verification.pro_case if verification else finding.rationale,
|
||||
)
|
||||
response = await client.complete(messages=[Message(role="user", content=prompt)], tools=[])
|
||||
verdict = parse_judge_response(_text(response.content))
|
||||
base = _panel_a(finding, verification) if role == "panel_a" else _panel_b(finding, verification)
|
||||
plausible = str(verdict.get("verdict", "")) == "plausible"
|
||||
confidence = (
|
||||
float(verdict.get("confidence", base.confidence)) if isinstance(verdict.get("confidence"), int | float) else base.confidence
|
||||
)
|
||||
panels.append(
|
||||
PanelVerdict(
|
||||
role=role,
|
||||
leaning=leaning,
|
||||
disposition=base.disposition if plausible or role == "panel_a" else FusionDisposition.REJECTED,
|
||||
confidence=confidence,
|
||||
vulnerability_case=base.vulnerability_case,
|
||||
false_positive_case=base.false_positive_case,
|
||||
rationale=str(verdict.get("cause", base.rationale)),
|
||||
model_id=model,
|
||||
)
|
||||
)
|
||||
return panels
|
||||
|
||||
|
||||
def _text(content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(str(block.get("text", "")) if isinstance(block, dict) else str(block) for block in content)
|
||||
return str(content)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FusionDecision",
|
||||
"FusionDisposition",
|
||||
"PanelVerdict",
|
||||
"fuse_finding",
|
||||
"fuse_findings_dispatch",
|
||||
"fuse_findings",
|
||||
"fusion_triggers",
|
||||
"should_fuse",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user