mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
chore(bench): archive reference-only scripts
A1-A3 of the legacy inventory, moved with git mv to benchmarks/archive/: the stage-2 verifier chain (verify_batched, score_batched, verify_sinks, model_sinks, sink_candidates), the v2 tool hunter (hunt.py) and the harnessx-removal equality proof. The plane's verify/agree/roles tasks replaced the first six; the harnessx-removal spec is closed. - The six independent scripts keep HERE = benchmarks/independent (population, evaluate.py, results) and add their own directory for archived siblings. - harnessx_removal_equality's REPO root is one level deeper. - tests/test_plane_scoring_parity.py loads score_batched from the archive (ran here, not skipped); test_harnessx_removal_equality points at the new path. - benchmarks/archive/README.md: what each script was, which committed records cite it, the commit that last ran it. paths.md maps old -> new; committed records are not rewritten. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bd74ba63b2
commit
067adb9845
@@ -12,7 +12,7 @@ import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
_SPEC = importlib.util.spec_from_file_location("hxr_equality", ROOT / "benchmarks" / "harnessx_removal_equality.py")
|
||||
_SPEC = importlib.util.spec_from_file_location("hxr_equality", ROOT / "benchmarks" / "archive" / "harnessx_removal_equality.py")
|
||||
assert _SPEC is not None and _SPEC.loader is not None
|
||||
equality = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(equality)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""`agree` scores exactly as the reference measurement did (ai-service-plane Requirement 6).
|
||||
|
||||
The reference is `benchmarks/independent/score_batched.py` over the batched-check scans: sites anchored by
|
||||
The reference is `benchmarks/archive/score_batched.py` over the batched-check scans: sites anchored by
|
||||
`score_batched.anchored`, matched by `evaluate.matches_v2` against `evaluate.hunks(case, "old")`. The plane cannot
|
||||
ship the benchmark tree, so the generator resolves the ranges into `case.json` and `agree.matches_declared`
|
||||
reproduces the rule. These tests run both on the recorded reference outputs of the 15 validation cases and
|
||||
@@ -23,6 +23,7 @@ from openultrasast.plane.manifests import load_manifests
|
||||
from openultrasast.plane.tasks import agree
|
||||
|
||||
BENCH = Path("benchmarks/independent")
|
||||
ARCHIVE = Path("benchmarks/archive") # score_batched.py, archived 2026-10-02
|
||||
POPULATION = BENCH / "population-v2.toml"
|
||||
SCANS = Path.home() / "ousast-results" / "independent-v2-batched-check" / "scans"
|
||||
REPOS = Path.home() / ".cache" / "openultrasast" / "independent"
|
||||
@@ -33,10 +34,10 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
def _load(name: str) -> ModuleType:
|
||||
def _load(name: str, where: Path = BENCH) -> ModuleType:
|
||||
sys.path.insert(0, str(BENCH.resolve())) # score_batched imports evaluate by name
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location(f"parity_{name}", BENCH / f"{name}.py")
|
||||
spec = importlib.util.spec_from_file_location(f"parity_{name}", where / f"{name}.py")
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
@@ -47,7 +48,7 @@ def _load(name: str) -> ModuleType:
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def reference() -> tuple[ModuleType, ModuleType]:
|
||||
evaluate, score_batched = _load("evaluate"), _load("score_batched")
|
||||
evaluate, score_batched = _load("evaluate"), _load("score_batched", ARCHIVE)
|
||||
evaluate.use(POPULATION.resolve())
|
||||
return evaluate, score_batched
|
||||
|
||||
|
||||
Reference in New Issue
Block a user