From 346cc703c3bde6ace33b24f864038eeb809f5d40 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 6 Sep 2026 11:45:00 +0200 Subject: [PATCH] test(learning-harness): a flaky harness must not be mistaken for a flaky detector The round-zero test decided which materialized side it was looking at by searching the whole path for a substring, and the random temporary directory name sometimes contained it. The detector then "found" the bug on the fixed side too, the pair flipped, and the model comparison failed about one run in eight. The check is now the directory name alone. This matters beyond the test: a noise floor exists to measure how much a detector disagrees with itself, and noise coming from the harness would be recorded as the detector's own and inflate every family's regression budget. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- tests/test_learning_rounds.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py index 92be6ba..eb8dd12 100644 --- a/tests/test_learning_rounds.py +++ b/tests/test_learning_rounds.py @@ -55,13 +55,24 @@ def _finding(family: str) -> StaticFinding: ) +def _is_vulnerable_side(root: Path) -> bool: + """The materialized side, by directory name only. + + An earlier version also matched "-v" anywhere in the path, which the random temporary directory name + sometimes contained, so the detector "found" the bug on the fixed side too and the pair flipped. A + flaky harness is indistinguishable from a flaky detector, which is precisely what a noise floor must + not measure. + """ + return root.name == "vuln" + + def _steady(root: Path) -> list[StaticFinding]: - return [_finding("injection")] if "-v" in str(root) or root.name == "vuln" else [] + return [_finding("injection")] if _is_vulnerable_side(root) else [] def _flaky(counter: list[int]): # type: ignore[no-untyped-def] def scan(root: Path) -> list[StaticFinding]: - if not ("-v" in str(root) or root.name == "vuln"): + if not _is_vulnerable_side(root): return [] counter.append(1) return [_finding("injection")] if len(counter) % 2 else []