mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
refactor(harnessx-removal): retire the HarnessX scan paths (task 5)
The standard scan now runs run_hunter_pool, verify_findings and the deterministic fuse_findings directly; the HarnessX hunter orchestrator, the LLM judge dispatcher (verify_judge.py, deleted with its test) and the LLM fusion panels are gone, and so are their extra-absent degradations. host_under_harnessx is dropped from stage_processors. config: [models] verifier and [fusion] panel_model/decider_model raise RetiredConfigError naming the plane replacement (the CLI prints it and exits 2); a [harnessx] section loads with one warning and is otherwise ignored. harness.json keeps "verifier": null. Tests that asserted the fallback are renamed to the behaviour they assert; the module-audit manifest drops verify_judge. Nothing outside the two modules deleted in task 6 imports harness_ext or hunter_harness. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
b68fe27c74
commit
cbadc38d5e
+17
-63
@@ -36,14 +36,12 @@ from .calibration import (
|
||||
)
|
||||
from .complexity import map as complexity_map
|
||||
from .complexity.ledger import persist_verdicts
|
||||
from .config import ModelLayerConfig, ObligationsConfig, ResolvedConfig, load_config, load_dotenv
|
||||
from .config import ModelLayerConfig, ObligationsConfig, ResolvedConfig, RetiredConfigError, load_config, load_dotenv
|
||||
from .findings import StaticFinding, quick_scan_findings, write_findings
|
||||
from .fusion import FusionDecision, fuse_findings_dispatch
|
||||
from .fusion import FusionDecision, fuse_findings
|
||||
from .gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||
from .harness_ext import has_harnessx
|
||||
from .hunter import run_hunter_pool, write_hunter_trajectories
|
||||
from .hunter_harness import HxScanOrchestrator
|
||||
from .improve import RoundOutcome, run_improvement
|
||||
from .index import build_code_chunks
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
||||
@@ -91,7 +89,6 @@ from .semantic import (
|
||||
from .semantic.facts import FactLoadError, SemanticFacts
|
||||
from .stages import Stage, plan_for_mode, record_completed, record_skip, skip_as_degradation, stages_payload
|
||||
from .verification import VerificationResult, verify_findings, write_verification_results
|
||||
from .verify_judge import verify_findings_dispatch
|
||||
|
||||
CALIBRATION_DIR = ".openultrasast/calibration"
|
||||
|
||||
@@ -125,6 +122,14 @@ def _configure_logging() -> None:
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
try:
|
||||
return _main(argv)
|
||||
except RetiredConfigError as exc: # a retired key is an error that names its replacement, exit 2
|
||||
print(f"ousast: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def _main(argv: list[str] | None) -> int:
|
||||
if (argv if argv is not None else sys.argv[1:])[:1] != ["pre-push"]:
|
||||
load_dotenv()
|
||||
parser = argparse.ArgumentParser(prog="ousast")
|
||||
@@ -431,13 +436,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
|
||||
plan = plan_for_mode(mode)
|
||||
config = load_config(config_path if config_path.exists() else None)
|
||||
# Capability gates for the optional HarnessX agentic plane. When the extra is
|
||||
# absent (default/CI), both stay False and the deterministic path runs unchanged.
|
||||
hunter_model = config.models.hunter
|
||||
verifier_model = config.models.verifier
|
||||
harnessx_present = has_harnessx()
|
||||
hx_hunter = mode == "standard" and bool(hunter_model) and harnessx_present
|
||||
hx_verify = mode == "standard" and bool(verifier_model) and harnessx_present
|
||||
run = create_scan_run(path, config)
|
||||
runtime = HarnessRuntime(
|
||||
scan_id=run.scan_id,
|
||||
@@ -469,28 +468,10 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
write_rankings(rankings, run.root / "rank" / "ranking.json")
|
||||
write_ranking_calibrations(calibrations, run.root / "calibration" / "applied_calibrations.json")
|
||||
if mode == "standard":
|
||||
if hx_hunter and hunter_model:
|
||||
orchestrator = HxScanOrchestrator(
|
||||
provider_model=hunter_model,
|
||||
provider=config.harnessx.provider,
|
||||
max_cost_usd=config.harnessx.max_cost_usd,
|
||||
token_threshold=config.harnessx.token_threshold,
|
||||
)
|
||||
hunter_result = runtime.run_stage(
|
||||
"hunter_pool",
|
||||
lambda: orchestrator.run_pool(
|
||||
run.target, targets, rankings, scan_id=run.scan_id, ruleset=ruleset, policy=policy, emit=runtime.emit
|
||||
),
|
||||
)
|
||||
else:
|
||||
if bool(hunter_model) and not harnessx_present:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "hunter_pool", "requested": "harnessx", "reason": "harnessx_extra_unavailable", "fallback": "run_hunter_pool"}
|
||||
)
|
||||
hunter_result = runtime.run_stage(
|
||||
"hunter_pool",
|
||||
lambda: run_hunter_pool(run.target, targets, rankings, scan_id=run.scan_id, ruleset=ruleset, policy=policy),
|
||||
)
|
||||
hunter_result = runtime.run_stage(
|
||||
"hunter_pool",
|
||||
lambda: run_hunter_pool(run.target, targets, rankings, scan_id=run.scan_id, ruleset=ruleset, policy=policy),
|
||||
)
|
||||
findings = hunter_result.findings
|
||||
trajectories = hunter_result.trajectories
|
||||
else:
|
||||
@@ -530,46 +511,19 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
write_findings(findings, findings_path)
|
||||
if trajectories:
|
||||
write_hunter_trajectories(trajectories, trajectories_path)
|
||||
if mode == "standard" and bool(verifier_model) and not harnessx_present:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "verify", "requested": "harnessx", "reason": "harnessx_extra_unavailable", "fallback": "structural_verifier"}
|
||||
)
|
||||
verifications = runtime.run_stage(
|
||||
"verify",
|
||||
lambda: verify_findings_dispatch(
|
||||
findings,
|
||||
verifier_model=verifier_model,
|
||||
verifier_provider=config.harnessx.provider,
|
||||
use_harnessx=hx_verify,
|
||||
),
|
||||
)
|
||||
verifications = runtime.run_stage("verify", lambda: verify_findings(findings))
|
||||
write_verification_results(verifications, verification_path)
|
||||
runtime.run_stage(
|
||||
"record_calibration",
|
||||
lambda: _persist_calibration_feedback(run, ledger_path, prior_learnings, findings, verifications),
|
||||
)
|
||||
# Fusion: two-panel adjudication for triggered findings (standard mode). Runs
|
||||
# deterministically by default; routes panels through the configured provider when
|
||||
# a panel model is set and the extra is present, else falls back + records a degradation.
|
||||
# Fusion: deterministic two-panel adjudication for triggered findings (standard mode).
|
||||
fusion_decisions: list[FusionDecision] = []
|
||||
fusion_path = run.root / "fusion.json"
|
||||
if mode == "standard" and config.fusion.enabled:
|
||||
hx_fusion = bool(config.fusion.panel_model) and harnessx_present
|
||||
if bool(config.fusion.panel_model) and not harnessx_present:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "fusion", "requested": "harnessx", "reason": "harnessx_extra_unavailable", "fallback": "deterministic_panels"}
|
||||
)
|
||||
fusion_decisions = runtime.run_stage(
|
||||
"fusion",
|
||||
lambda: fuse_findings_dispatch(
|
||||
findings,
|
||||
verifications,
|
||||
panel_model=config.fusion.panel_model,
|
||||
decider_model=config.fusion.decider_model,
|
||||
provider=config.harnessx.provider,
|
||||
use_harnessx=hx_fusion,
|
||||
high_assurance=config.fusion.high_assurance,
|
||||
),
|
||||
lambda: fuse_findings(findings, verifications, high_assurance=config.fusion.high_assurance),
|
||||
)
|
||||
if fusion_decisions:
|
||||
fusion_path.write_text(json.dumps([decision.to_dict() for decision in fusion_decisions], indent=2, sort_keys=True) + "\n")
|
||||
|
||||
+39
-31
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tomllib
|
||||
from dataclasses import asdict, dataclass
|
||||
@@ -11,6 +12,41 @@ from openultrasast.contracts import Contract
|
||||
|
||||
DEFAULT_VECTOR_STORE = "json-local"
|
||||
|
||||
# Retirement notes (harnessx-removal Req 1.3, 3.3). A key that asked for a retired LLM capability fails and names
|
||||
# the replacement, never a silent downgrade; the old budget section only warns. Every line below that names the
|
||||
# removed plane carries its retirement date, which the reference-search test holds the rest of the tree to.
|
||||
_RETIRED_ON = "retired 2026-09-30 with HarnessX"
|
||||
_PLANE_RUN = "`ousast plane run`, see ops/ax/README.md"
|
||||
_RETIRED_PANELS = (
|
||||
f"LLM fusion panels were retired; fusion is deterministic; independent LLM agreement is the plane's `agree` task ({_PLANE_RUN})"
|
||||
)
|
||||
RETIRED_KEYS: dict[tuple[str, str], str] = {
|
||||
("models", "verifier"): f"LLM verification runs on the plane (`verify` + `agree`, {_PLANE_RUN})",
|
||||
("fusion", "panel_model"): _RETIRED_PANELS,
|
||||
("fusion", "decider_model"): _RETIRED_PANELS,
|
||||
}
|
||||
RETIRED_SECTION = "harnessx" # retired 2026-09-30: loads with one warning and is otherwise ignored (Req 3.3)
|
||||
RETIRED_SECTION_WARNING = (
|
||||
"`[harnessx]` is ignored (retired 2026-09-30): HarnessX was removed; agentic work runs on the ax plane "
|
||||
"with per-task budgets, see ops/ax/README.md"
|
||||
)
|
||||
|
||||
|
||||
class RetiredConfigError(ValueError):
|
||||
"""A config key asked for a capability that was retired; the message names its replacement."""
|
||||
|
||||
|
||||
def _check_retired(data: dict[str, object]) -> None:
|
||||
problems = []
|
||||
for (section, key), replacement in RETIRED_KEYS.items():
|
||||
table = data.get(section)
|
||||
if isinstance(table, dict) and key in table:
|
||||
problems.append(f"`[{section}] {key}` was {_RETIRED_ON}: {replacement}. Remove the key.")
|
||||
if problems:
|
||||
raise RetiredConfigError("\n".join(problems))
|
||||
if RETIRED_SECTION in data:
|
||||
logging.getLogger("openultrasast.config").warning(RETIRED_SECTION_WARNING)
|
||||
|
||||
|
||||
def load_dotenv(path: Path | None = None, *, force: bool = False) -> None:
|
||||
"""Load KEY=VALUE lines from .env into os.environ. Stdlib only; never overrides."""
|
||||
@@ -34,7 +70,6 @@ def load_dotenv(path: Path | None = None, *, force: bool = False) -> None:
|
||||
class ModelConfig:
|
||||
ranker: str | None = None
|
||||
hunter: str | None = None
|
||||
verifier: str | None = None
|
||||
patcher: str | None = None
|
||||
judge: str | None = None # second, independent judgement before anything is published as proven
|
||||
chat_base_url: str | None = None # chat endpoint; independent of the embedding endpoint
|
||||
@@ -95,17 +130,6 @@ class RulesetConfig:
|
||||
min_emit_precision: float = 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessxConfig:
|
||||
# Optional agentic plane (the openultrasast[harnessx] extra). `provider`
|
||||
# selects the HarnessX LLM provider used by both the hunter pool and the
|
||||
# llm-judge verifier; the budget caps bound per-task cost and token spend.
|
||||
# Inert unless the extra is installed and a model is configured in [models].
|
||||
provider: str = "anthropic"
|
||||
max_cost_usd: float = 2.0
|
||||
token_threshold: int = 120_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HardeningConfig:
|
||||
# Release-readiness controls. redact_secrets masks credentials in traces/reports;
|
||||
@@ -116,12 +140,9 @@ class HardeningConfig:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FusionConfig:
|
||||
# Two-panel fusion adjudication for triggered findings (standard mode). Runs
|
||||
# deterministically by default; setting panel_model routes the panels through
|
||||
# the configured HarnessX provider. high_assurance forces fusion on every finding.
|
||||
# Two-panel fusion adjudication for triggered findings (standard mode). Deterministic
|
||||
# panels and reconciler; high_assurance forces fusion on every finding.
|
||||
enabled: bool = True
|
||||
panel_model: str | None = None
|
||||
decider_model: str | None = None
|
||||
high_assurance: bool = False
|
||||
|
||||
|
||||
@@ -198,7 +219,6 @@ class ResolvedConfig:
|
||||
static_analysis: StaticAnalysisConfig = StaticAnalysisConfig()
|
||||
score: ScoreConfig = ScoreConfig()
|
||||
ruleset: RulesetConfig = RulesetConfig()
|
||||
harnessx: HarnessxConfig = HarnessxConfig()
|
||||
fusion: FusionConfig = FusionConfig()
|
||||
hardening: HardeningConfig = HardeningConfig()
|
||||
complexity: ComplexityConfig = ComplexityConfig()
|
||||
@@ -217,6 +237,7 @@ def load_config(config_path: Path | None = None, *, dotenv: bool = True) -> Reso
|
||||
if config_path is not None and config_path.exists():
|
||||
with config_path.open("rb") as handle:
|
||||
data = tomllib.load(handle)
|
||||
_check_retired(data)
|
||||
|
||||
return ResolvedConfig(
|
||||
models=_load_models(data.get("models", {})),
|
||||
@@ -227,7 +248,6 @@ def load_config(config_path: Path | None = None, *, dotenv: bool = True) -> Reso
|
||||
static_analysis=_load_static_analysis(data.get("static_analysis", {})),
|
||||
score=_load_score(data.get("score", {})),
|
||||
ruleset=_load_ruleset_config(data.get("ruleset", {})),
|
||||
harnessx=_load_harnessx(data.get("harnessx", {})),
|
||||
fusion=_load_fusion(data.get("fusion", {})),
|
||||
hardening=_load_hardening(data.get("hardening", {})),
|
||||
complexity=_load_complexity(data.get("complexity", {})),
|
||||
@@ -263,7 +283,6 @@ def _load_models(value: object) -> ModelConfig:
|
||||
return ModelConfig(
|
||||
ranker=_string(data.get("ranker")),
|
||||
hunter=_string(data.get("hunter")),
|
||||
verifier=_string(data.get("verifier")),
|
||||
patcher=_string(data.get("patcher")),
|
||||
judge=_string(data.get("judge")),
|
||||
chat_base_url=_string(data.get("chat_base_url")),
|
||||
@@ -342,15 +361,6 @@ def _load_ruleset_config(value: object) -> RulesetConfig:
|
||||
)
|
||||
|
||||
|
||||
def _load_harnessx(value: object) -> HarnessxConfig:
|
||||
data = _section(value)
|
||||
return HarnessxConfig(
|
||||
provider=_string(data.get("provider")) or "anthropic",
|
||||
max_cost_usd=_float_value(data.get("max_cost_usd"), 2.0),
|
||||
token_threshold=_int_value(data.get("token_threshold"), 120_000),
|
||||
)
|
||||
|
||||
|
||||
def _load_hardening(value: object) -> HardeningConfig:
|
||||
data = _section(value)
|
||||
return HardeningConfig(
|
||||
@@ -363,8 +373,6 @@ def _load_fusion(value: object) -> FusionConfig:
|
||||
data = _section(value)
|
||||
return FusionConfig(
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
panel_model=_string(data.get("panel_model")),
|
||||
decider_model=_string(data.get("decider_model")),
|
||||
high_assurance=bool(data.get("high_assurance", False)),
|
||||
)
|
||||
|
||||
|
||||
+9
-137
@@ -7,11 +7,9 @@ disclosure, or an explicit high-assurance request. Two panels independently revi
|
||||
bounded evidence (one steel-manning the vulnerability case, one the false-positive
|
||||
case), critique and revise, vote, and a decider issues the final disposition.
|
||||
|
||||
The deterministic engine here is the zero-dependency core (and the fallback). When the
|
||||
``openultrasast[harnessx]`` extra and panel models are configured, ``fuse_findings_dispatch``
|
||||
routes the panels through real LLM reviews; the decider reconciliation stays deterministic
|
||||
and auditable. Every fused finding receives exactly one of the five dispositions, and the
|
||||
decision discloses model IDs, panel roles, votes, degradations, and the decision source.
|
||||
The panels and the decider are deterministic and auditable. Every fused finding receives
|
||||
exactly one of the five dispositions, and the decision discloses panel roles, votes and the
|
||||
decision source. Independent LLM agreement is the plane's ``agree`` task, not this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,7 +19,6 @@ from dataclasses import asdict, dataclass, field
|
||||
from enum import StrEnum
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import has_harnessx
|
||||
from .verification import VerificationResult, VerificationStatus
|
||||
|
||||
|
||||
@@ -57,7 +54,7 @@ class FusionDecision:
|
||||
triggered: bool
|
||||
triggers: list[str]
|
||||
disposition: FusionDisposition
|
||||
decision_source: str # "deterministic-reconciler" | "llm-panels" | "no-fusion"
|
||||
decision_source: str # "deterministic-reconciler" | "no-fusion"
|
||||
panels: list[PanelVerdict]
|
||||
votes: dict[str, int]
|
||||
model_ids: list[str]
|
||||
@@ -191,155 +188,30 @@ def fuse_finding(
|
||||
)
|
||||
|
||||
|
||||
def fuse_findings_dispatch(
|
||||
def fuse_findings(
|
||||
findings: list[StaticFinding],
|
||||
verifications: list[VerificationResult],
|
||||
*,
|
||||
panel_model: str | None = None,
|
||||
decider_model: str | None = None,
|
||||
provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
high_assurance: bool = False,
|
||||
mitigated_ids: frozenset[str] = frozenset(),
|
||||
) -> list[FusionDecision]:
|
||||
"""Fuse every finding that the trigger policy selects.
|
||||
|
||||
Deterministic by default. When ``use_harnessx`` is set, a panel model is configured,
|
||||
and the extra is present, the panels are produced by real LLM reviews; otherwise the
|
||||
deterministic panels run and (if an LLM run was requested) a degradation is recorded.
|
||||
"""
|
||||
"""Fuse every finding that the trigger policy selects, with the deterministic panels."""
|
||||
verification_by_id = {result.finding_id: result for result in verifications}
|
||||
want_llm = use_harnessx and bool(panel_model)
|
||||
decisions: list[FusionDecision] = []
|
||||
for finding in findings:
|
||||
verification = verification_by_id.get(finding.finding_id)
|
||||
triggers = fusion_triggers(finding, verification, high_assurance=high_assurance)
|
||||
if not triggers:
|
||||
continue
|
||||
if want_llm and has_harnessx():
|
||||
decisions.append(_fuse_with_llm_panels(finding, verification, triggers, panel_model, decider_model, provider, mitigated_ids))
|
||||
else:
|
||||
decision = fuse_finding(finding, verification, triggers, mitigated_ids=mitigated_ids)
|
||||
if want_llm and not has_harnessx():
|
||||
decision = _with_degradation(decision, "fusion_llm_unavailable: harnessx extra absent; used deterministic panels")
|
||||
decisions.append(decision)
|
||||
if triggers:
|
||||
decisions.append(fuse_finding(finding, verification, triggers, mitigated_ids=mitigated_ids))
|
||||
return decisions
|
||||
|
||||
|
||||
def _with_degradation(decision: FusionDecision, note: str) -> FusionDecision:
|
||||
return FusionDecision(
|
||||
finding_id=decision.finding_id,
|
||||
triggered=decision.triggered,
|
||||
triggers=decision.triggers,
|
||||
disposition=decision.disposition,
|
||||
decision_source=decision.decision_source,
|
||||
panels=decision.panels,
|
||||
votes=decision.votes,
|
||||
model_ids=decision.model_ids,
|
||||
degradations=[*decision.degradations, note],
|
||||
warnings=decision.warnings,
|
||||
)
|
||||
|
||||
|
||||
def _fuse_with_llm_panels(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
triggers: list[str],
|
||||
panel_model: str | None,
|
||||
decider_model: str | None,
|
||||
provider: str,
|
||||
mitigated_ids: frozenset[str],
|
||||
) -> FusionDecision:
|
||||
"""LLM-backed panels with a deterministic, auditable decider (lazy/guarded)."""
|
||||
import asyncio
|
||||
|
||||
model = panel_model or ""
|
||||
try:
|
||||
panels = asyncio.run(_run_llm_panels(finding, verification, model, provider))
|
||||
source = "llm-panels"
|
||||
warnings: list[str] = []
|
||||
except Exception as exc: # noqa: BLE001 — panel/provider failure falls back, never crashes the scan
|
||||
panels = [_panel_a(finding, verification), _panel_b(finding, verification)]
|
||||
source = "deterministic-reconciler"
|
||||
warnings = [f"llm_panels_failed: {type(exc).__name__}"]
|
||||
disposition, _rationale = _reconcile(finding, verification, panels, mitigated_ids)
|
||||
votes = Counter(str(panel.disposition) for panel in panels)
|
||||
model_ids = sorted({panel.model_id for panel in panels if panel.model_id})
|
||||
if decider_model:
|
||||
model_ids = sorted({*model_ids, decider_model})
|
||||
return FusionDecision(
|
||||
finding_id=finding.finding_id,
|
||||
triggered=True,
|
||||
triggers=triggers,
|
||||
disposition=disposition,
|
||||
decision_source=source,
|
||||
panels=panels,
|
||||
votes=dict(sorted(votes.items())),
|
||||
model_ids=model_ids,
|
||||
degradations=[],
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def _run_llm_panels(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
model: str,
|
||||
provider: str,
|
||||
) -> list[PanelVerdict]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
|
||||
from .harness_ext import build_provider
|
||||
|
||||
client = build_provider(model, provider)
|
||||
panels: list[PanelVerdict] = []
|
||||
for role, leaning, stance in (
|
||||
("panel_a", "vulnerability", "Argue the vulnerability case as strongly as the evidence allows."),
|
||||
("panel_b", "false_positive", "Argue the false-positive case as strongly as the evidence allows."),
|
||||
):
|
||||
summary = f"reachability={finding.reachability_status}; severity={finding.severity}; tags={', '.join(finding.tags) or 'none'}"
|
||||
prompt = build_judge_prompt(
|
||||
task_description=f"{stance} Finding: {finding.title} at {finding.path}:{finding.line}.",
|
||||
trajectory_summary=summary,
|
||||
extracted_answer=verification.pro_case if verification else finding.rationale,
|
||||
)
|
||||
response = await client.complete(messages=[Message(role="user", content=prompt)], tools=[])
|
||||
verdict = parse_judge_response(_text(response.content))
|
||||
base = _panel_a(finding, verification) if role == "panel_a" else _panel_b(finding, verification)
|
||||
plausible = str(verdict.get("verdict", "")) == "plausible"
|
||||
confidence = (
|
||||
float(verdict.get("confidence", base.confidence)) if isinstance(verdict.get("confidence"), int | float) else base.confidence
|
||||
)
|
||||
panels.append(
|
||||
PanelVerdict(
|
||||
role=role,
|
||||
leaning=leaning,
|
||||
disposition=base.disposition if plausible or role == "panel_a" else FusionDisposition.REJECTED,
|
||||
confidence=confidence,
|
||||
vulnerability_case=base.vulnerability_case,
|
||||
false_positive_case=base.false_positive_case,
|
||||
rationale=str(verdict.get("cause", base.rationale)),
|
||||
model_id=model,
|
||||
)
|
||||
)
|
||||
return panels
|
||||
|
||||
|
||||
def _text(content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(str(block.get("text", "")) if isinstance(block, dict) else str(block) for block in content)
|
||||
return str(content)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FusionDecision",
|
||||
"FusionDisposition",
|
||||
"PanelVerdict",
|
||||
"fuse_finding",
|
||||
"fuse_findings_dispatch",
|
||||
"fuse_findings",
|
||||
"fusion_triggers",
|
||||
"should_fuse",
|
||||
]
|
||||
|
||||
@@ -180,7 +180,7 @@ def write_harness_config(
|
||||
model_roles={
|
||||
"ranker": config.models.ranker,
|
||||
"hunter": config.models.hunter,
|
||||
"verifier": config.models.verifier,
|
||||
"verifier": None, # schema kept byte for byte; LLM verification runs on the plane
|
||||
"patcher": config.models.patcher,
|
||||
},
|
||||
processor_versions={processor.spec.name: processor.spec.version for processor in processors},
|
||||
|
||||
@@ -4,14 +4,9 @@ The deterministic scan stages carry their work in named *slots* and run with mod
|
||||
invocation disabled (``skip_model = True``). Each stage declares a read/write slot
|
||||
allow-list that is validated on every lifecycle hook, preserving the
|
||||
``ProcessorSpec.reads/writes`` provenance discipline of the sync ``HarnessRuntime``
|
||||
(the top-level driver) — without forcing the deterministic stages through HarnessX's
|
||||
agent loop ("bridge, do not map", per the spec design).
|
||||
(the top-level driver).
|
||||
|
||||
This is the zero-dependency substrate. When the optional ``openultrasast[harnessx]``
|
||||
extra is present, the same model-disabled processors can be hosted under a HarnessX
|
||||
``MultiHookProcessor`` (see ``stage_processors.host_under_harnessx``); the
|
||||
``skip_model`` marker maps onto HarnessX's ``BeforeModelEvent.skip_model`` so no
|
||||
deterministic stage incurs a model call.
|
||||
This is the zero-dependency substrate for the deterministic pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -3,12 +3,8 @@
|
||||
Each stage is a :class:`slot_contract.SlotContractMixin` with ``skip_model = True`` and
|
||||
a declared read/write slot allow-list. Running them through a :class:`SlotPipeline`
|
||||
reproduces the deterministic quick-scan path byte-for-byte while enforcing the slot
|
||||
contract on every lifecycle hook. The hunter and verifier stages are *not* here —
|
||||
they are the agentic plane (``hunter_harness``/``verify_judge``); this module is the
|
||||
deterministic, model-free remainder.
|
||||
|
||||
``host_under_harnessx`` is the optional, lazy bridge: it wraps a model-disabled stage
|
||||
as a HarnessX ``MultiHookProcessor`` that disables the model call on ``before_model``.
|
||||
contract on every lifecycle hook. The hunter and verifier stages are *not* here; this
|
||||
module is the deterministic, model-free remainder that ``test_gate`` pins.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -126,36 +122,6 @@ def run_quick_pipeline(
|
||||
return build_deterministic_pipeline().run(initial, runtime=runtime)
|
||||
|
||||
|
||||
def host_under_harnessx(processor: SlotContractMixin) -> Any:
|
||||
"""Host a model-disabled deterministic stage under a HarnessX ``MultiHookProcessor``.
|
||||
|
||||
Lazy + capability-guarded (imports ``harnessx`` only when called). On the
|
||||
``before_model`` hook the hosted stage yields a ``BeforeModelEvent`` with
|
||||
``skip_model=True``, so no deterministic stage incurs a model call when it runs on
|
||||
the HarnessX runloop. Raises :class:`HarnessXUnavailableError` when the extra is absent.
|
||||
"""
|
||||
import dataclasses
|
||||
|
||||
from .harness_ext import require_harnessx
|
||||
|
||||
require_harnessx()
|
||||
from harnessx.core.events import BeforeModelEvent
|
||||
from harnessx.core.processor import MultiHookProcessor
|
||||
|
||||
class _ModelDisabledStage(MultiHookProcessor): # type: ignore[misc, valid-type]
|
||||
_singleton_group = "ousast_deterministic"
|
||||
_order = 50
|
||||
|
||||
def __init__(self, stage: SlotContractMixin) -> None:
|
||||
self.stage = stage
|
||||
self.name = stage.name
|
||||
|
||||
async def on_before_model(self, event: BeforeModelEvent): # type: ignore[no-untyped-def]
|
||||
yield dataclasses.replace(event, skip_model=True, synthetic_output=f"[deterministic stage {self.stage.name}: model disabled]")
|
||||
|
||||
return _ModelDisabledStage(processor)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EntryPointProcessor",
|
||||
"PolicyScoringProcessor",
|
||||
@@ -165,6 +131,5 @@ __all__ = [
|
||||
"ScoreProcessor",
|
||||
"build_deterministic_pipeline",
|
||||
"deterministic_processors",
|
||||
"host_under_harnessx",
|
||||
"run_quick_pipeline",
|
||||
]
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
"""Verifier dispatcher: HarnessX llm-judge when available, structural fallback otherwise.
|
||||
|
||||
The default path (no extra) is the existing :func:`verification.verify_findings`
|
||||
verbatim — same ``list[VerificationResult]``, same rejection of unknown reachability —
|
||||
so behaviour is byte-identical without HarnessX. The judge path is lazy/guarded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import build_provider, has_harnessx
|
||||
from .verification import (
|
||||
EvidenceLevel,
|
||||
VerificationResult,
|
||||
VerificationStatus,
|
||||
build_independent_verifier_context,
|
||||
is_report_verified,
|
||||
verify_findings,
|
||||
)
|
||||
|
||||
|
||||
def verify_findings_dispatch(
|
||||
findings: list[StaticFinding],
|
||||
*,
|
||||
verifier_model: str | None = None,
|
||||
verifier_provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
) -> list[VerificationResult]:
|
||||
"""Drop-in superset of :func:`verification.verify_findings`.
|
||||
|
||||
Falls back to the structural verifier unless the HarnessX extra is present, a
|
||||
verifier model is configured, and ``use_harnessx`` is set.
|
||||
"""
|
||||
if not (use_harnessx and verifier_model and has_harnessx()):
|
||||
return verify_findings(findings)
|
||||
return _verify_with_judge(findings, verifier_model, verifier_provider)
|
||||
|
||||
|
||||
def _verify_with_judge(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
import asyncio
|
||||
|
||||
return asyncio.run(_judge_all(findings, verifier_model, verifier_provider))
|
||||
|
||||
|
||||
async def _judge_all(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
|
||||
provider = build_provider(verifier_model, verifier_provider)
|
||||
results: list[VerificationResult] = []
|
||||
for finding in findings:
|
||||
prompt = build_judge_prompt(
|
||||
task_description=f"Is this a real, exploitable security issue: {finding.title}? Judge on evidence only.",
|
||||
trajectory_summary=_finding_summary(finding),
|
||||
extracted_answer=finding.rationale,
|
||||
)
|
||||
try:
|
||||
response = await provider.complete(messages=[Message(role="user", content=prompt)], tools=[])
|
||||
verdict = parse_judge_response(_text(response.content))
|
||||
except Exception: # noqa: BLE001 — judge/provider failure must never auto-reject a finding
|
||||
verdict = {"verdict": "no_answer", "confidence": 0.0}
|
||||
results.append(_verdict_to_verification(finding, verdict))
|
||||
return results
|
||||
|
||||
|
||||
def _verdict_to_verification(finding: StaticFinding, verdict: dict[str, object]) -> VerificationResult:
|
||||
"""Map an llm-judge verdict onto a VerificationResult (infra failure never auto-rejects)."""
|
||||
label = str(verdict.get("verdict", "no_answer"))
|
||||
raw_confidence = verdict.get("confidence", 0.0)
|
||||
confidence = 0.0
|
||||
if isinstance(raw_confidence, (int, float)):
|
||||
confidence = float(raw_confidence)
|
||||
elif isinstance(raw_confidence, str):
|
||||
try:
|
||||
confidence = float(raw_confidence)
|
||||
except ValueError:
|
||||
confidence = 0.0
|
||||
cause = str(verdict.get("cause", "")).strip()
|
||||
lesson = str(verdict.get("lesson", "")).strip()
|
||||
missing = str(verdict.get("missing", "")).strip() or _missing_capability_summary(verdict)
|
||||
level = EvidenceLevel(finding.evidence_level)
|
||||
|
||||
if label == "plausible" and confidence >= 0.6:
|
||||
status = VerificationStatus.ACCEPTED
|
||||
elif label in ("unsupported", "refused"):
|
||||
status = VerificationStatus.REJECTED
|
||||
else: # hedging / format_wrong / no_answer / low-confidence plausible
|
||||
status = VerificationStatus.NEEDS_EVIDENCE
|
||||
|
||||
if status == VerificationStatus.ACCEPTED:
|
||||
pro_case = cause or "The judge found the issue plausible on the available evidence."
|
||||
counter_case = "Impact and exploitability still require deeper verifier or dynamic evidence."
|
||||
else:
|
||||
pro_case = "A candidate issue exists from the finding's evidence."
|
||||
counter_case = cause or "The judge could not confirm the issue from the available evidence."
|
||||
|
||||
return VerificationResult(
|
||||
finding_id=finding.finding_id,
|
||||
status=status,
|
||||
evidence_level=level,
|
||||
verified=is_report_verified(level, status),
|
||||
pro_case=pro_case,
|
||||
counter_case=counter_case,
|
||||
tie_breaker=lesson or "Confirm attacker control, sanitizer absence, and impact.",
|
||||
required_next_step=missing or "Collect corroborating evidence before reporting as verified.",
|
||||
context_sources=sorted(build_independent_verifier_context(finding).keys()),
|
||||
)
|
||||
|
||||
|
||||
def _finding_summary(finding: StaticFinding) -> str:
|
||||
return (
|
||||
f"{finding.title} at {finding.path}:{finding.line} (function={finding.function_name or 'unknown'}); "
|
||||
f"reachability={finding.reachability_status}; tags={', '.join(finding.tags) or 'none'}"
|
||||
)
|
||||
|
||||
|
||||
def _missing_capability_summary(verdict: dict[str, object]) -> str:
|
||||
capability = verdict.get("missing_capability")
|
||||
if isinstance(capability, dict):
|
||||
return str(capability.get("summary", "")).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _text(content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [str(block.get("text", "")) if isinstance(block, dict) else str(block) for block in content]
|
||||
return "".join(parts)
|
||||
return str(content)
|
||||
Reference in New Issue
Block a user