From adb70b6b982f02ca5e423c04b246ee331d38a1d6 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:39:27 +0200 Subject: [PATCH] chore(tests): drop the HarnessX equality test that guards an archived one-off Legacy inventory 2, item R2. tests/test_harnessx_removal_equality.py loaded benchmarks/archive/harnessx_removal_equality.py by path and checked that its comparison sees a difference. The spec is implementation-complete and the proof is the committed pair of records (2026-09-30-harnessx-removal-{baseline,equality}); the archive README says these scripts are reference only and a record is re-run at the commit it names, not with today's package. A unit test that keeps the archived copy importable contradicted that and would break the next time cli or learn.cli move. The archive README row now says the test was dropped. test_removed_plane_references.py (the Req 2.4 name search) is a different test and stays. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- benchmarks/archive/README.md | 2 +- tests/test_harnessx_removal_equality.py | 65 ------------------------- 2 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 tests/test_harnessx_removal_equality.py diff --git a/benchmarks/archive/README.md b/benchmarks/archive/README.md index 357341f..2b72129 100644 --- a/benchmarks/archive/README.md +++ b/benchmarks/archive/README.md @@ -16,7 +16,7 @@ maps each old path to the new one. The records themselves are not rewritten. | `model_sinks.py` | Stage-1 source-only model classification of sinks. Moved into the package as `plane/tasks/roles.py`. | `independent/selection-v2-model-sinks.json`, `measurements/2026-10-01-decision-engine-harvest/record.json`, both `2026-10-01-decision-engine-leak-audit` records | `05d5699` (named in `selection-v2-model-sinks.json`) | | `sink_candidates.py` | Diagnostic: vocabulary-declared sink candidates per case. Its declaration patterns live on in `plane/tasks/repo_facts.py`. | `independent/results-v2-model-pipeline.json` ("vocabulary sink-first: 2/17") | no commit recorded | | `hunt.py` | The v2 tool-hunter run over the population (protocol `hunter-protocol-v2.md`, including its `FAMILY_WORDS` table). Replaced by the plane's `verify` / `agree` tasks. | `independent/hunter-protocol-v2.md`, `independent/results-v2-hunter.json` (a label source in `learn/sources.toml`) | `6bfb05f`, the commit that introduced the protocol, which pins the runner | -| `harnessx_removal_equality.py` | One-off equality proof for `harnessx-removal` (record and compare deterministic outputs). The spec is closed. `tests/test_harnessx_removal_equality.py` still checks that the comparison detects differences. | `measurements/2026-09-30-harnessx-removal-baseline/`, `measurements/2026-09-30-harnessx-removal-equality/` | baseline `ca5caf7`, candidate `b9afb66` | +| `harnessx_removal_equality.py` | One-off equality proof for `harnessx-removal` (record and compare deterministic outputs). The spec is closed; its unit test was dropped on 2026-10-02 so the archived copy is reference only like the rest. | `measurements/2026-09-30-harnessx-removal-baseline/`, `measurements/2026-09-30-harnessx-removal-equality/` | baseline `ca5caf7`, candidate `b9afb66` | The six `independent` scripts still read the population, `evaluate.py` and the results from `benchmarks/independent/`. Their `HERE` points there, so outputs land where they always did. diff --git a/tests/test_harnessx_removal_equality.py b/tests/test_harnessx_removal_equality.py deleted file mode 100644 index 55f63bd..0000000 --- a/tests/test_harnessx_removal_equality.py +++ /dev/null @@ -1,65 +0,0 @@ -"""The equality instrument (harnessx-removal design section 3) sees a difference when there is one. - -A comparison that always says "equal" would prove nothing, so these tests feed it records that differ in one -place each and require it to name that place, and records that differ only in excluded fields and require it -to pass. -""" - -from __future__ import annotations - -import importlib.util -import json -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -_SPEC = importlib.util.spec_from_file_location("hxr_equality", ROOT / "benchmarks" / "archive" / "harnessx_removal_equality.py") -assert _SPEC is not None and _SPEC.loader is not None -equality = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(equality) - - -def _record(path: Path, *, findings: object, suite: dict[str, str], stdout: str = "findings=3\n") -> Path: - (path / "scan").mkdir(parents=True) - (path / "suite.json").write_text(equality.dump(suite)) - (path / "scan" / "findings.json").write_text(equality.dump(findings)) - (path / "scan" / "stdout.txt").write_text(stdout) - (path / "env.json").write_text(json.dumps({"tree_ish": str(path)})) - return path - - -def _normalized(raw: dict[str, object], root: str) -> object: - return equality.normalize(raw, root, "/tmp/x") - - -def test_excluded_fields_and_run_ids_do_not_count(tmp_path: Path) -> None: - a = _normalized({"rule": "sqli", "runtime_seconds": 1.2, "path": "/r1/runs/20260930T101010Z-0123abcd/f.py", "t_ms": 4}, "/r1") - b = _normalized({"rule": "sqli", "runtime_seconds": 9.9, "path": "/r2/runs/20261001T111111Z-ffffffff/f.py", "t_ms": 7}, "/r2") - suite = {"tests.test_x::test_a": "passed"} - diffs = equality.compare(_record(tmp_path / "a", findings=a, suite=suite), _record(tmp_path / "b", findings=b, suite=suite), {}) - assert diffs == [] - - -def test_a_changed_finding_is_reported_with_its_key(tmp_path: Path) -> None: - suite = {"tests.test_x::test_a": "passed"} - a = _record(tmp_path / "a", findings=[{"rule": "sqli", "line": 3}], suite=suite) - b = _record(tmp_path / "b", findings=[{"rule": "sqli", "line": 4}], suite=suite) - assert equality.compare(a, b, {}) == ["scan/findings.json: $[0].line: 3 -> 4"] - assert equality.main(["compare", str(a), str(b)]) == 1 - - -def test_a_changed_outcome_or_unmapped_test_is_reported(tmp_path: Path) -> None: - a = _record(tmp_path / "a", findings=[], suite={"t::a": "passed", "t::gone": "passed", "t::old": "skipped"}) - b = _record(tmp_path / "b", findings=[], suite={"t::a": "failure", "t::new": "skipped", "t::extra": "passed"}, stdout="findings=2\n") - diffs = equality.compare(a, b, {"deleted": ["t::gone"], "renamed": {"t::old": "t::new"}}) - assert diffs == [ - "suite.json: t::a passed -> failure", - "suite.json: t::extra new in candidate", - "scan/stdout.txt: first differing line 1: 'findings=3' -> 'findings=2'", - ] - - -def test_a_file_present_on_one_side_only_is_reported(tmp_path: Path) -> None: - a = _record(tmp_path / "a", findings=[], suite={}) - b = _record(tmp_path / "b", findings=[], suite={}) - (b / "scan" / "fusion.json").write_text("[]\n") - assert equality.compare(a, b, {}) == ["scan/fusion.json: present only in candidate"]