diff --git a/benchmarks/archive/README.md b/benchmarks/archive/README.md index 357341f..2b72129 100644 --- a/benchmarks/archive/README.md +++ b/benchmarks/archive/README.md @@ -16,7 +16,7 @@ maps each old path to the new one. The records themselves are not rewritten. | `model_sinks.py` | Stage-1 source-only model classification of sinks. Moved into the package as `plane/tasks/roles.py`. | `independent/selection-v2-model-sinks.json`, `measurements/2026-10-01-decision-engine-harvest/record.json`, both `2026-10-01-decision-engine-leak-audit` records | `05d5699` (named in `selection-v2-model-sinks.json`) | | `sink_candidates.py` | Diagnostic: vocabulary-declared sink candidates per case. Its declaration patterns live on in `plane/tasks/repo_facts.py`. | `independent/results-v2-model-pipeline.json` ("vocabulary sink-first: 2/17") | no commit recorded | | `hunt.py` | The v2 tool-hunter run over the population (protocol `hunter-protocol-v2.md`, including its `FAMILY_WORDS` table). Replaced by the plane's `verify` / `agree` tasks. | `independent/hunter-protocol-v2.md`, `independent/results-v2-hunter.json` (a label source in `learn/sources.toml`) | `6bfb05f`, the commit that introduced the protocol, which pins the runner | -| `harnessx_removal_equality.py` | One-off equality proof for `harnessx-removal` (record and compare deterministic outputs). The spec is closed. `tests/test_harnessx_removal_equality.py` still checks that the comparison detects differences. | `measurements/2026-09-30-harnessx-removal-baseline/`, `measurements/2026-09-30-harnessx-removal-equality/` | baseline `ca5caf7`, candidate `b9afb66` | +| `harnessx_removal_equality.py` | One-off equality proof for `harnessx-removal` (record and compare deterministic outputs). The spec is closed; its unit test was dropped on 2026-10-02 so the archived copy is reference only like the rest. | `measurements/2026-09-30-harnessx-removal-baseline/`, `measurements/2026-09-30-harnessx-removal-equality/` | baseline `ca5caf7`, candidate `b9afb66` | The six `independent` scripts still read the population, `evaluate.py` and the results from `benchmarks/independent/`. Their `HERE` points there, so outputs land where they always did. diff --git a/tests/test_harnessx_removal_equality.py b/tests/test_harnessx_removal_equality.py deleted file mode 100644 index 55f63bd..0000000 --- a/tests/test_harnessx_removal_equality.py +++ /dev/null @@ -1,65 +0,0 @@ -"""The equality instrument (harnessx-removal design section 3) sees a difference when there is one. - -A comparison that always says "equal" would prove nothing, so these tests feed it records that differ in one -place each and require it to name that place, and records that differ only in excluded fields and require it -to pass. -""" - -from __future__ import annotations - -import importlib.util -import json -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -_SPEC = importlib.util.spec_from_file_location("hxr_equality", ROOT / "benchmarks" / "archive" / "harnessx_removal_equality.py") -assert _SPEC is not None and _SPEC.loader is not None -equality = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(equality) - - -def _record(path: Path, *, findings: object, suite: dict[str, str], stdout: str = "findings=3\n") -> Path: - (path / "scan").mkdir(parents=True) - (path / "suite.json").write_text(equality.dump(suite)) - (path / "scan" / "findings.json").write_text(equality.dump(findings)) - (path / "scan" / "stdout.txt").write_text(stdout) - (path / "env.json").write_text(json.dumps({"tree_ish": str(path)})) - return path - - -def _normalized(raw: dict[str, object], root: str) -> object: - return equality.normalize(raw, root, "/tmp/x") - - -def test_excluded_fields_and_run_ids_do_not_count(tmp_path: Path) -> None: - a = _normalized({"rule": "sqli", "runtime_seconds": 1.2, "path": "/r1/runs/20260930T101010Z-0123abcd/f.py", "t_ms": 4}, "/r1") - b = _normalized({"rule": "sqli", "runtime_seconds": 9.9, "path": "/r2/runs/20261001T111111Z-ffffffff/f.py", "t_ms": 7}, "/r2") - suite = {"tests.test_x::test_a": "passed"} - diffs = equality.compare(_record(tmp_path / "a", findings=a, suite=suite), _record(tmp_path / "b", findings=b, suite=suite), {}) - assert diffs == [] - - -def test_a_changed_finding_is_reported_with_its_key(tmp_path: Path) -> None: - suite = {"tests.test_x::test_a": "passed"} - a = _record(tmp_path / "a", findings=[{"rule": "sqli", "line": 3}], suite=suite) - b = _record(tmp_path / "b", findings=[{"rule": "sqli", "line": 4}], suite=suite) - assert equality.compare(a, b, {}) == ["scan/findings.json: $[0].line: 3 -> 4"] - assert equality.main(["compare", str(a), str(b)]) == 1 - - -def test_a_changed_outcome_or_unmapped_test_is_reported(tmp_path: Path) -> None: - a = _record(tmp_path / "a", findings=[], suite={"t::a": "passed", "t::gone": "passed", "t::old": "skipped"}) - b = _record(tmp_path / "b", findings=[], suite={"t::a": "failure", "t::new": "skipped", "t::extra": "passed"}, stdout="findings=2\n") - diffs = equality.compare(a, b, {"deleted": ["t::gone"], "renamed": {"t::old": "t::new"}}) - assert diffs == [ - "suite.json: t::a passed -> failure", - "suite.json: t::extra new in candidate", - "scan/stdout.txt: first differing line 1: 'findings=3' -> 'findings=2'", - ] - - -def test_a_file_present_on_one_side_only_is_reported(tmp_path: Path) -> None: - a = _record(tmp_path / "a", findings=[], suite={}) - b = _record(tmp_path / "b", findings=[], suite={}) - (b / "scan" / "fusion.json").write_text("[]\n") - assert equality.compare(a, b, {}) == ["scan/fusion.json: present only in candidate"]