mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
chore(harnessx-removal): delete HarnessX code and packaging (task 6)
Deletes harness_ext.py and hunter_harness.py with their HarnessX-only tests (the judge path, verify_judge.py, went in task 5). The harnessx extra, its mypy override and its uv.lock entries are removed (uv lock drops only packages; no remaining version changes). The conftest fixture becomes assert_cold_import; test_gate's two never-import checks become one allow-list (stdlib, openultrasast, yaml) over the governance planes' module-level imports, and the skipped live-extra tolerance test is deleted. Audit rows for the two modules are removed, stage_processors is reworded, and the module-audit manifest is regenerated (no orphan). test_removed_plane_references implements Req 2.4 over src, tests and pyproject.toml: a match must carry a `retired 2026-..` tag (the spec's own slug is not counted). The README part is a strict xfail until task 8 rewrites the README's plane sections. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
cbadc38d5e
commit
b9afb66250
+5
-12
@@ -5,12 +5,6 @@ import sys
|
||||
|
||||
import pytest
|
||||
|
||||
_HARNESSX_GUARD = (
|
||||
"\nimport sys as _sys\n"
|
||||
"_leaked = sorted(m for m in _sys.modules if m == 'harnessx' or m.startswith('harnessx.'))\n"
|
||||
"assert not _leaked, 'leaked harnessx modules: ' + repr(_leaked)\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _pointer_pairs_offline(monkeypatch: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory) -> None:
|
||||
@@ -31,17 +25,16 @@ def _repos_offline(monkeypatch: pytest.MonkeyPatch, tmp_path_factory: pytest.Tem
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_cold_of_harnessx(): # type: ignore[no-untyped-def]
|
||||
"""Run ``code`` in a fresh interpreter and assert it imported no ``harnessx`` module.
|
||||
def assert_cold_import(): # type: ignore[no-untyped-def]
|
||||
"""Run ``code`` in a fresh interpreter and assert it exits 0.
|
||||
|
||||
Hermetic by construction: the lazy-import guarantee is verified the same way
|
||||
whether or not the optional extra is installed in this environment (a global
|
||||
``sys.modules`` check would be polluted by other tests once the extra is present).
|
||||
Hermetic by construction: an import-laziness check (``code`` asserts on ``sys.modules`` itself) is not
|
||||
polluted by modules other tests have already imported into this process.
|
||||
"""
|
||||
|
||||
def _check(code: str) -> str:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-c", code + _HARNESSX_GUARD],
|
||||
[sys.executable, "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
+29
-30
@@ -2,15 +2,15 @@
|
||||
|
||||
8.1 — the recall/FP gate is a hard constraint a score improvement cannot unblock.
|
||||
8.2 — default-ruleset benchmark output is byte-identical to a committed baseline.
|
||||
8.3 — a HarnessX-backed stage must stay within tolerance of the zero-dep baseline.
|
||||
8.4 — the governance/scoring/benchmark planes never import the HarnessX extra.
|
||||
8.3 — the tolerance band a changed stage is held to against the zero-dep baseline.
|
||||
8.4 — the governance/scoring/benchmark planes import only stdlib, the package and PyYAML at module scope.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.gate import (
|
||||
FALSE_POSITIVE_CEILING,
|
||||
RECALL_FLOOR,
|
||||
@@ -20,7 +20,6 @@ from openultrasast.gate import (
|
||||
merge_gate,
|
||||
within_tolerance,
|
||||
)
|
||||
from openultrasast.harness_ext import has_harnessx
|
||||
|
||||
GOLDEN = Path("tests/golden/detection_baseline.json")
|
||||
|
||||
@@ -108,7 +107,7 @@ def test_benchmark_output_is_byte_identical_to_golden_baseline() -> None:
|
||||
)
|
||||
|
||||
|
||||
# ---- 8.3 HarnessX-backed stage stays within detection tolerance -------------
|
||||
# ---- 8.3 detection tolerance band -------------------------------------------
|
||||
|
||||
|
||||
def test_within_tolerance_band_logic() -> None:
|
||||
@@ -121,15 +120,6 @@ def test_within_tolerance_band_logic() -> None:
|
||||
assert not within_tolerance(base, DetectionMetrics(100, 93, 100, 10), recall_tolerance=0.05, fp_tolerance=0.05)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="HarnessX extra not installed; live agent run unavailable offline")
|
||||
def test_harnessx_stage_stays_within_tolerance_of_baseline() -> None: # pragma: no cover - needs extra + live keys
|
||||
# When the extra and provider keys are present, a HarnessX-backed hunter run is
|
||||
# compared against the zero-dep baseline and must stay within tolerance. Live
|
||||
# LLM execution is unavailable in CI, so this gates on the capability.
|
||||
baseline = detection_gate(evaluate_corpus()).overall
|
||||
assert within_tolerance(baseline, baseline)
|
||||
|
||||
|
||||
# ---- 8.4 zero-dependency guard ----------------------------------------------
|
||||
|
||||
GOVERNANCE_SCORING_BENCHMARK_PLANES = (
|
||||
@@ -152,24 +142,33 @@ GOVERNANCE_SCORING_BENCHMARK_PLANES = (
|
||||
)
|
||||
|
||||
|
||||
def test_governance_scoring_benchmark_planes_never_import_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# Hermetic: a fresh interpreter imports every plane and must pull no harnessx.
|
||||
imports = "\n".join(f"import {module}" for module in GOVERNANCE_SCORING_BENCHMARK_PLANES)
|
||||
assert_cold_of_harnessx(imports)
|
||||
_ALLOWED_TOP_LEVEL = frozenset(sys.stdlib_module_names) | {"openultrasast", "yaml"}
|
||||
|
||||
|
||||
def test_planes_have_no_module_level_harnessx_import() -> None:
|
||||
# Flags only column-0 (module-level) imports; an indented lazy import inside a
|
||||
# capability-guarded function (e.g. stage_processors.host_under_harnessx) is fine.
|
||||
def _module_level_imports(tree: ast.Module) -> list[str]:
|
||||
"""Top-level names imported outside any function or class body (a lazy import inside a function is fine)."""
|
||||
names: list[str] = []
|
||||
pending: list[ast.AST] = list(tree.body)
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
|
||||
continue
|
||||
if isinstance(node, ast.Import):
|
||||
names.extend(alias.name.split(".")[0] for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
names.append("openultrasast" if node.level else (node.module or "").split(".")[0])
|
||||
else:
|
||||
pending.extend(ast.iter_child_nodes(node))
|
||||
return names
|
||||
|
||||
|
||||
def test_governance_planes_import_only_stdlib_and_declared_dependencies() -> None:
|
||||
# An allow-list, not a deny-list: any optional extra at module scope fails without being named here.
|
||||
src = Path("src/openultrasast")
|
||||
offenders = []
|
||||
for module in GOVERNANCE_SCORING_BENCHMARK_PLANES:
|
||||
rel = module.removeprefix("openultrasast.").replace(".", "/")
|
||||
candidates = [src / f"{rel}.py", src / rel / "__init__.py"]
|
||||
path = next((c for c in candidates if c.exists()), None)
|
||||
if path is None:
|
||||
continue
|
||||
for line in path.read_text().splitlines():
|
||||
if line.startswith(("import harnessx", "from harnessx")):
|
||||
offenders.append(f"{path}: {line}")
|
||||
assert offenders == [], f"plane modules import HarnessX at module scope: {offenders}"
|
||||
path = next((c for c in (src / f"{rel}.py", src / rel / "__init__.py") if c.exists()), None)
|
||||
assert path is not None, f"{module} vanished from the guard's list"
|
||||
offenders += [f"{module}: {name}" for name in _module_level_imports(ast.parse(path.read_text())) if name not in _ALLOWED_TOP_LEVEL]
|
||||
assert offenders == [], f"governance planes import beyond stdlib, openultrasast and yaml at module scope: {offenders}"
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
"""HarnessX optional-extra packaging + capability guard (Phase 3 task 5.2)."""
|
||||
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.harness_ext import (
|
||||
HARNESSX_EXTRA,
|
||||
SUPPORTED_PROVIDERS,
|
||||
HarnessXUnavailableError,
|
||||
build_provider,
|
||||
has_harnessx,
|
||||
require_harnessx,
|
||||
)
|
||||
|
||||
PYPROJECT = tomllib.loads(Path("pyproject.toml").read_text())
|
||||
|
||||
|
||||
def test_core_dependencies_are_pyyaml_only() -> None:
|
||||
assert PYPROJECT["project"]["dependencies"] == ["pyyaml>=6.0"] # ai-service-plane task 1: manifests are YAML; HarnessX stays an extra
|
||||
|
||||
|
||||
def test_harnessx_is_a_sha_pinned_optional_extra() -> None:
|
||||
extras = PYPROJECT["project"]["optional-dependencies"]["harnessx"]
|
||||
assert len(extras) == 1
|
||||
assert re.fullmatch(
|
||||
r"harnessx @ git\+https://github\.com/Darwin-Agent/HarnessX\.git@[0-9a-f]{40}",
|
||||
extras[0],
|
||||
), extras[0]
|
||||
|
||||
|
||||
def test_importing_openultrasast_does_not_import_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# The big modules (cli, harness_ext) must not eagerly import the heavy extra.
|
||||
assert_cold_of_harnessx("import openultrasast.cli\nimport openultrasast.harness_ext")
|
||||
|
||||
|
||||
def test_has_harnessx_probes_without_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
assert isinstance(has_harnessx(), bool)
|
||||
# find_spec must not import the module.
|
||||
assert_cold_of_harnessx("from openultrasast.harness_ext import has_harnessx\nassert isinstance(has_harnessx(), bool)")
|
||||
|
||||
|
||||
def test_require_harnessx_raises_with_install_guidance_when_absent() -> None:
|
||||
if has_harnessx():
|
||||
pytest.skip("HarnessX extra is installed in this environment")
|
||||
with pytest.raises(HarnessXUnavailableError, match=re.escape(HARNESSX_EXTRA)):
|
||||
require_harnessx()
|
||||
|
||||
|
||||
def test_build_provider_rejects_unknown_provider_before_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# Name is validated before any import, so this fails loudly even without the extra.
|
||||
with pytest.raises(ValueError, match="unknown HarnessX provider"):
|
||||
build_provider("gpt-4o", "bogus")
|
||||
assert_cold_of_harnessx(
|
||||
"from openultrasast.harness_ext import build_provider\n"
|
||||
"try:\n build_provider('gpt-4o', 'bogus')\n"
|
||||
"except ValueError:\n pass\n"
|
||||
"else:\n raise SystemExit('expected ValueError')\n"
|
||||
)
|
||||
|
||||
|
||||
def test_supported_providers_cover_anthropic_and_openai() -> None:
|
||||
assert {"anthropic", "openai"} <= set(SUPPORTED_PROVIDERS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("provider", "class_name"), [("anthropic", "AnthropicProvider"), ("openai", "OpenAIProvider")])
|
||||
def test_build_provider_constructs_selected_provider(provider: str, class_name: str) -> None:
|
||||
if not has_harnessx():
|
||||
pytest.skip("HarnessX extra not installed in this environment")
|
||||
built = build_provider("test-model", provider)
|
||||
assert type(built).__name__ == class_name
|
||||
@@ -1,44 +0,0 @@
|
||||
"""HarnessX hunter sub-harness: guarding, findings parsing, construction smoke (task 6.1)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.harness_ext import HarnessXUnavailableError, build_sast, has_harnessx
|
||||
from openultrasast.hunter_harness import HxScanOrchestrator, _extract_findings
|
||||
|
||||
|
||||
def test_module_import_does_not_pull_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
assert_cold_of_harnessx("import openultrasast.hunter_harness")
|
||||
|
||||
|
||||
def test_extract_findings_parses_json_array_and_tolerates_garbage() -> None:
|
||||
text = 'Here are the findings:\n[{"line": 9, "title": "SQLi", "cwe": "CWE-89", "rationale": "concat"}]\nDone.'
|
||||
items = _extract_findings(text)
|
||||
assert items == [{"line": 9, "title": "SQLi", "cwe": "CWE-89", "rationale": "concat"}]
|
||||
assert _extract_findings("no findings here") == []
|
||||
assert _extract_findings("[not, valid, json]") == []
|
||||
|
||||
|
||||
@pytest.mark.skipif(has_harnessx(), reason="exercises the extra-absent guard")
|
||||
def test_build_sast_raises_without_extra() -> None:
|
||||
with pytest.raises(HarnessXUnavailableError):
|
||||
build_sast()
|
||||
|
||||
|
||||
@pytest.mark.skipif(has_harnessx(), reason="exercises the extra-absent guard")
|
||||
def test_orchestrator_construction_raises_without_extra() -> None:
|
||||
with pytest.raises(HarnessXUnavailableError):
|
||||
HxScanOrchestrator(provider_model="claude-sonnet-4-6")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="construction smoke; requires the harnessx extra")
|
||||
def test_build_sast_returns_harness_config_offline() -> None:
|
||||
from harnessx.core.harness import HarnessConfig
|
||||
|
||||
assert isinstance(build_sast(), HarnessConfig)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="construction smoke; requires the harnessx extra")
|
||||
def test_orchestrator_builds_agent_offline() -> None:
|
||||
# Constructs the model + harness without any LLM call (provider client is lazy).
|
||||
agent = HxScanOrchestrator(provider_model="claude-sonnet-4-6")._build_agent()
|
||||
assert type(agent).__name__ == "Harness"
|
||||
@@ -25,13 +25,15 @@ def test_every_module_is_classified() -> None:
|
||||
|
||||
|
||||
def test_the_named_subsystems_each_carry_an_explicit_decision() -> None:
|
||||
"""Req 3.3: the five older-phase subsystems get a decision, not silence."""
|
||||
"""Req 3.3: the older-phase subsystems still in the tree get a decision, not silence."""
|
||||
from openultrasast.model.audit import audit
|
||||
|
||||
rows = {row.module: row for row in audit(SRC)}
|
||||
for name in ("fusion", "mcp", "skills", "harness_ext", "hunter_harness"):
|
||||
for name in ("fusion", "mcp", "skills"):
|
||||
assert name in rows, f"{name} vanished without a recorded decision"
|
||||
assert rows[name].reason, f"{name} has no stated reason"
|
||||
# The removed agentic plane's modules are gone from the tree, not carried as rows (harnessx-removal Req 2.1).
|
||||
assert not {"harness_ext", "hunter_harness", "verify_judge"} & set(rows)
|
||||
|
||||
|
||||
def test_the_manifest_matches_the_tree() -> None:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Requirement 2.4 of the removal spec: the removed agentic plane is named only in its retirement notes.
|
||||
|
||||
A case-insensitive search of ``src/``, ``tests/``, ``pyproject.toml`` and the README may match only lines tagged
|
||||
``retired 2026-..``: the retired-key table and the one warning in ``config.py``, the retirement tests, and the README's
|
||||
plane section. The spec's own slug (its directory name, the equality instrument's file name) is a citation of the
|
||||
removal, not a use of the plane, and is not counted. The pattern is built at runtime so this file does not match it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
_NAME = "harness" + "x"
|
||||
_PATTERN = re.compile(_NAME, re.IGNORECASE)
|
||||
_SPEC_SLUG = re.compile(_NAME + r"[-_]removal", re.IGNORECASE)
|
||||
_TAG = re.compile(r"retired 2026-\d\d-\d\d", re.IGNORECASE)
|
||||
|
||||
|
||||
def _untagged(paths: list[Path]) -> list[str]:
|
||||
offenders = []
|
||||
for path in paths:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (UnicodeDecodeError, OSError):
|
||||
continue
|
||||
for number, line in enumerate(text.splitlines(), 1):
|
||||
if _PATTERN.search(_SPEC_SLUG.sub("", line)) and not _TAG.search(line):
|
||||
offenders.append(f"{path.relative_to(ROOT)}:{number}: {line.strip()[:120]}")
|
||||
return offenders
|
||||
|
||||
|
||||
def _tree(*roots: str) -> list[Path]:
|
||||
return sorted(p for root in roots for p in (ROOT / root).rglob("*") if p.is_file() and "__pycache__" not in p.parts)
|
||||
|
||||
|
||||
def test_no_harnessx_outside_retirement_notes() -> None: # the Req 2.4 check itself, retired 2026-09-30
|
||||
offenders = _untagged([*_tree("src", "tests"), ROOT / "pyproject.toml"])
|
||||
assert offenders == [], "the removed plane is named outside a tagged retirement note:\n" + "\n".join(offenders)
|
||||
|
||||
|
||||
def test_the_retirement_notes_exist() -> None:
|
||||
"""The search is not vacuous: the one warning and the retired keys are where the design puts them."""
|
||||
config = (ROOT / "src/openultrasast/config.py").read_text()
|
||||
assert _PATTERN.search(config) and _TAG.search(config)
|
||||
|
||||
|
||||
@pytest.mark.xfail(strict=True, reason="the README's plane sections are rewritten in task 8; drop this mark there")
|
||||
def test_the_readme_names_the_removed_plane_only_in_its_retirement_note() -> None:
|
||||
offenders = _untagged([ROOT / "README.md"])
|
||||
assert offenders == [], "\n".join(offenders)
|
||||
@@ -25,14 +25,14 @@ def test_core_dependencies_are_pyyaml_only_and_semantic_is_optional() -> None:
|
||||
assert "semantic:" in " ".join(payload["tool"]["pytest"]["ini_options"]["markers"])
|
||||
|
||||
|
||||
def test_cli_import_does_not_load_tree_sitter(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
def test_cli_import_does_not_load_tree_sitter(assert_cold_import) -> None: # type: ignore[no-untyped-def]
|
||||
proc_code = (
|
||||
"import sys\n"
|
||||
"import openultrasast.cli\n"
|
||||
"leaked = sorted(m for m in sys.modules if m == 'tree_sitter' or m.startswith('tree_sitter_'))\n"
|
||||
"assert not leaked, leaked\n"
|
||||
)
|
||||
assert_cold_of_harnessx(proc_code)
|
||||
assert_cold_import(proc_code)
|
||||
|
||||
|
||||
def test_probe_off_yields_no_javascript_grammar(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
Reference in New Issue
Block a user