mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
fe45081b43
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.
Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.
Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 90.0%
leak 12.3% 6.7%
detection -- 96.7%
cost ~$0.69 $0.0068
27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).
One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.
The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
96 lines
3.8 KiB
Python
96 lines
3.8 KiB
Python
"""model-grounded-detection Req 1.3: nothing survives that imports what this feature removed.
|
|
|
|
The noise architecture and the mechanism/evolve loop are deleted, not deprecated. This walks the AST of every
|
|
surviving module and fails on any import that names one of them — the check that turns a five-thousand-line
|
|
deletion from crash-driven discovery into one assertion.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
from pathlib import Path
|
|
|
|
SRC = Path("src/openultrasast")
|
|
|
|
REMOVED_MODULES = frozenset(
|
|
{
|
|
# the noise architecture (the whole learning/ package; taxonomy, candidates and endpoint re-homed to model/)
|
|
"learning",
|
|
# the mechanism/evolve optimisation loop
|
|
"semantic.mechanisms",
|
|
"semantic.variants",
|
|
"semantic.variant_search",
|
|
"semantic.prove_budget",
|
|
"semantic.loo",
|
|
"semantic.seed",
|
|
}
|
|
)
|
|
|
|
|
|
def _module_names(tree: ast.AST) -> set[str]:
|
|
names: set[str] = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.module:
|
|
names.add(node.module)
|
|
elif isinstance(node, ast.Import):
|
|
names.update(alias.name for alias in node.names)
|
|
return names
|
|
|
|
|
|
def _names_a_removed_module(imported: str) -> bool:
|
|
trimmed = imported.lstrip(".")
|
|
return any(trimmed == removed or trimmed.startswith(f"{removed}.") for removed in REMOVED_MODULES)
|
|
|
|
|
|
def test_no_surviving_module_imports_a_removed_one() -> None:
|
|
offenders: dict[str, list[str]] = {}
|
|
for path in sorted(SRC.rglob("*.py")):
|
|
named = sorted(name for name in _module_names(ast.parse(path.read_text())) if _names_a_removed_module(name))
|
|
if named:
|
|
offenders[str(path)] = named
|
|
assert not offenders, f"dangling imports of removed modules: {offenders}"
|
|
|
|
|
|
def test_the_removed_modules_are_gone_from_the_tree() -> None:
|
|
assert not (SRC / "learning").exists(), "the learning package is removed; taxonomy/candidates/endpoint live under model/"
|
|
for relative in (
|
|
"semantic/mechanisms.py",
|
|
"semantic/variants.py",
|
|
"semantic/variant_search.py",
|
|
"semantic/prove_budget.py",
|
|
"semantic/loo.py",
|
|
"semantic/seed.py",
|
|
):
|
|
assert not (SRC / relative).exists(), f"{relative} still exists"
|
|
|
|
|
|
def test_the_mechanism_lever_is_gone_from_the_retained_improve_loop() -> None:
|
|
"""`improve/` survives with its lever removed, not deleted.
|
|
|
|
The design's Removed list named `improve/evolve.py` whole, but the file is a mix: the mechanism lever
|
|
(deleted here) and the deterministic rule-status loop that `ousast improve` still runs from benchmark
|
|
evidence, which has no LLM in it and is not what this feature supersedes. Deleting the file would have
|
|
taken a retained capability with it, so the lever is stripped and the loop stays — asserted by symbol so
|
|
the lever cannot creep back.
|
|
"""
|
|
evolve = (SRC / "improve" / "evolve.py").read_text()
|
|
validator = (SRC / "improve" / "validator.py").read_text()
|
|
for symbol in (
|
|
"MechanismEdit",
|
|
"MechanismStore",
|
|
"apply_mechanism_edits",
|
|
"evaluate_mechanism_profiles",
|
|
"propose_mechanism_edits",
|
|
"validate_mechanism",
|
|
"mechanism_store",
|
|
"mechanism_candidates",
|
|
):
|
|
assert symbol not in evolve, f"{symbol} survives in improve/evolve.py"
|
|
assert symbol not in validator, f"{symbol} survives in improve/validator.py"
|
|
assert "def run_improvement(" in evolve and "def propose_status_edits(" in evolve, "the rule-status loop must survive"
|
|
|
|
|
|
def test_the_re_homed_modules_are_where_the_model_layer_expects_them() -> None:
|
|
for relative in ("model/taxonomy.py", "model/candidates.py", "model/endpoint.py", "model/specs.py"):
|
|
assert (SRC / relative).is_file(), f"{relative} is missing"
|