diff --git a/benchmarks/independent/evaluate.py b/benchmarks/independent/evaluate.py index f118d8e..cbb47b0 100644 --- a/benchmarks/independent/evaluate.py +++ b/benchmarks/independent/evaluate.py @@ -8,9 +8,12 @@ question is an instrument failure and is retried once. `score` applies protocol-v1.md exactly -- matching, detection, fixed-side and benign false alerts, unanswered -- and draws the precision sample (every third pooled vulnerable-pin finding, or all of them if 15 or fewer). +Protocol v2 (protocol-v2.md, population-v2.toml) scans the vulnerable and fixed pins twice each and credits a +finding at a declared site as well; `--population population-v2.toml` selects it for both commands. + Usage: - python benchmarks/independent/evaluate.py run --source [--only ID,...] - python benchmarks/independent/evaluate.py score + python benchmarks/independent/evaluate.py run --source [--only ID,...] [--population F] + python benchmarks/independent/evaluate.py score [--population F] """ from __future__ import annotations @@ -37,7 +40,28 @@ def cases() -> list[dict]: return tomllib.loads(POPULATION.read_text())["case"] +def version() -> str: + return POPULATION.stem.removeprefix("population-") + + +def use(population: Path) -> None: + """Point the module at a population file and its own results directory.""" + global POPULATION, RESULTS + POPULATION = population if population.is_absolute() else HERE / population.name + RESULTS = Path.home() / "ousast-results" / f"independent-{version()}" + + def pins(case: dict) -> dict[str, str]: + if version() != "v1": + # Protocol v2: the vulnerable and fixed pins twice each, the benign pair once. + return { + "vulnerable_a": case["vulnerable"], + "vulnerable_b": case["vulnerable"], + "fixed_a": case["fixed"], + "fixed_b": case["fixed"], + "benign_base": case["benign"]["base"], + "benign_tip": case["benign"]["tip"], + } return { "vulnerable": case["vulnerable"], "fixed": case["fixed"], @@ -207,15 +231,109 @@ def score(args: argparse.Namespace) -> int: return 0 +def matches_v2(finding: dict, case: dict, ranges: dict[str, list[tuple[int, int]]]) -> bool: + """protocol-v2.md: v1's rule with the record's declared site functions, or a declared `path::function`.""" + path, _, rest = finding["site"].partition(":") + line_text, _, function = rest.partition(":") + if finding.get("family") != case["family"]: + return False + sites = set(case.get("sites", [])) + if f"{path}::{function}" in sites: + return True + if path not in ranges: + return False + if function in {site.partition("::")[2] for site in sites}: + return True + line = int(line_text) if line_text.isdigit() else -1 + return any(first - LINE_WINDOW <= line <= last + LINE_WINDOW for first, last in ranges[path]) + + +def stability(one: list[dict], two: list[dict]) -> tuple[int, int]: + """(findings in exactly one run, findings in either).""" + a, b = {f["site"] for f in one}, {f["site"] for f in two} + return len(a ^ b), len(a | b) + + +def score_v2() -> int: + per_case, pool, unstable_pool = [], [], 0 + changing, total_questions, total_completed = [0, 0], 0, 0 + for case in cases(): + path = lambda label: RESULTS / "scans" / f"{case['id']}--{label}.json" # noqa: E731 + scans = {label: json.loads(path(label).read_text()) for label in pins(case) if path(label).is_file()} + if len(scans) < 6: + per_case.append({"id": case["id"], "status": "incomplete", "scans": sorted(scans)}) + continue + old, new = hunks(case, "old"), hunks(case, "new") + va, vb = scans["vulnerable_a"].get("findings", []), scans["vulnerable_b"].get("findings", []) + fa, fb = scans["fixed_a"].get("findings", []), scans["fixed_b"].get("findings", []) + hit_a = {f["site"] for f in va if matches_v2(f, case, old)} + hit_b = {f["site"] for f in vb if matches_v2(f, case, old)} + fixed_alerts = sorted({f["site"] for f in fa + fb if matches_v2(f, case, new)}) + base_regions = {tuple(f["site"].split(":")[0::2]) for f in scans["benign_base"].get("findings", [])} + benign_alerts = [f["site"] for f in scans["benign_tip"].get("findings", []) if tuple(f["site"].split(":")[0::2]) not in base_regions] + site_files = {site.partition("::")[0] for site in case.get("sites", [])} + covered = set(old) | site_files + answered = any(r.split(":")[0] in covered for label in ("vulnerable_a", "vulnerable_b") for r in scans[label].get("completed_regions", [])) + diff_v, union_v = stability(va, vb) + diff_f, union_f = stability(fa, fb) + changing[0] += diff_v + diff_f + changing[1] += union_v + union_f + for scan in scans.values(): + total_questions += int(scan.get("questions") or 0) + total_completed += int(scan.get("completed") or 0) + detected = sorted(hit_a & hit_b) + per_case.append( + { + "id": case["id"], "family": case["family"], "language": case["language"], + "detected": bool(detected), "detected_sites": detected, + "unstable": bool(hit_a ^ hit_b) and not detected, "unstable_sites": sorted(hit_a ^ hit_b), + "unanswered": not (hit_a | hit_b) and not answered, + "instrument_errors": {label: s["instrument_error"] for label, s in scans.items() if "instrument_error" in s}, + "fixed_side_alerts": fixed_alerts, "benign_alerts": benign_alerts, + "stability": {"vulnerable": [diff_v, union_v], "fixed": [diff_f, union_f]}, + "completion": {label: [s.get("completed"), s.get("questions")] for label, s in scans.items()}, + "seconds": {label: s.get("seconds") for label, s in scans.items()}, + } + ) # fmt: skip + stable = {f["site"] for f in vb} + pool += [{"case": case["id"], **f} for f in va if f["site"] in stable] + unstable_pool += len({f["site"] for f in va} ^ stable) + pool.sort(key=lambda f: (f["case"], f["site"])) + sample = pool if len(pool) <= 15 else pool[0::3] + scored = [c for c in per_case if c.get("status") != "incomplete"] + detected = sum(c["detected"] for c in scored) + result = { + "protocol": "protocol-v2.md", + "cases_scored": len(scored), "cases_incomplete": [c["id"] for c in per_case if c.get("status") == "incomplete"], + "recall": [detected, len(scored)], "recall_wilson95": wilson(detected, len(scored)), + "unstable_detections": [c["id"] for c in scored if c["unstable"]], + "fixed_side_alerts": sum(len(c["fixed_side_alerts"]) for c in scored), + "benign_alerts": sum(len(c["benign_alerts"]) for c in scored), + "completion": [total_completed, total_questions], + "stability": changing, + "pooled_stable_vulnerable_findings": len(pool), "unstable_vulnerable_findings": unstable_pool, + "precision_sample_size": len(sample), + "cases": per_case, "precision_sample": sample, + } # fmt: skip + (RESULTS / "score.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({k: v for k, v in result.items() if k not in ("cases", "precision_sample")}, indent=1)) + return 0 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) runner = sub.add_parser("run") runner.add_argument("--source", type=Path, required=True, help="a frozen export of the analyzer (freeze_source.sh)") runner.add_argument("--only", default="") - sub.add_parser("score") + runner.add_argument("--population", type=Path, default=POPULATION) + scorer = sub.add_parser("score") + scorer.add_argument("--population", type=Path, default=POPULATION) args = parser.parse_args() - return run(args) if args.command == "run" else score(args) + use(args.population) + if args.command == "run": + return run(args) + return score(args) if version() == "v1" else score_v2() if __name__ == "__main__": diff --git a/benchmarks/independent/protocol-v2.md b/benchmarks/independent/protocol-v2.md new file mode 100644 index 0000000..38c3f93 --- /dev/null +++ b/benchmarks/independent/protocol-v2.md @@ -0,0 +1,83 @@ +# Evaluation protocol v2 for the independent population v2 (pre-registered) + +Written and committed on 2026-09-26, BEFORE the first scan of any case in `population-v2.toml`. It fixes what is +measured, how a result is scored and what is reported, so that no rule of the scoring can be chosen after seeing +an outcome. Changing it after the first scan makes the run exploratory, and it must then be reported as such. + +Differences from protocol v1, each decided by the maintainer before any v2 case was scanned, and each a lesson +of the v1 run rather than of any v2 case: + +- **Declared sites.** v1 credited a finding only in a file the fix changed. alerta and YesWiki (v1) execute their + SQL in a helper the fix does not touch, so a correct report where the query runs could never count. Each v2 + case therefore declares, from its advisory and before any scan, the `sites` (`path::function`) where the + vulnerable operation happens; a finding there is credited too. +- **Two runs.** The engine's path selection is not deterministic: on v1 retunes one verdict flipped in 2 of 4 + identical runs, and a PMPro finding came and went between runs. The vulnerable and the fixed pin are scanned + twice each, and the difference is reported. + +## What is under test + +- The analyzer at the commit that introduces the v2 evaluation runner, exported with + `benchmarks/push/freeze_source.sh`; its hash is recorded in the result. +- Per case, only the case's own family (`injection`, `untrusted_destination` or `config_secrets`), through the + production discovery and scan (`benchmarks/push/finding_dump.py`), unbounded region count (5,000), a 2,400 s + deadline per scan, one engine container at a time. +- Six scans per case: the vulnerable pin twice (runs a, b), the fixed pin twice (a, b), the benign base and the + benign tip once each. Each checkout is a `git archive` of the pin, with no project tooling run on it. + +## Scoring, per case + +A finding MATCHES the case when its family is the case family and either +1. its file is one the fix changes (`git diff --name-only vulnerable fixed`) and either its function is one the + record names in `sites`, or its line lies within 10 lines of a hunk the fix changes on that side (v1's rule); + or +2. its `path::function` equals one of the record's declared `sites` (`` for file scope). + +- **Detected** (recall): a matching finding in BOTH vulnerable runs. +- **Unstable detection**: a matching finding in exactly one vulnerable run. Reported; counts as NOT detected. +- **Fixed-side false alert**: a matching finding in EITHER fixed run. +- **Benign false alert**: any finding of the case family at the benign tip whose (file, function) has no finding + at the benign base. +- **Unanswered**: neither vulnerable run completed a question covering a file the fix changes or a declared site + file. It counts as NOT detected, and the reason is reported. A scan with no questions at all, or unreadable + input, is an instrument failure: it is rerun once, and if it fails again it is reported as a failure, never as + a clean result. + +## Stability + +Per pin with two runs, the findings present in exactly one run divided by the findings present in either is +reported, per case and pooled. It is not a gate; it says how far a single run can be trusted. + +## Precision + +All findings of the case family present in BOTH vulnerable runs, across the population, are pooled and sorted by +(case, site). Every third one, starting from the first, is adjudicated against the source, as are all of them if +the pool holds 15 or fewer. Findings present in only one run are counted and reported, not adjudicated. The +standard is v1's: + +- **True**: attacker-controlled data reaches the sink without a sufficient guard. +- **True, privileged**: the same, reachable only with a privileged role. +- **False, guarded**: a guard makes the flow safe. +- **False, not attacker**: the value does not come from an attacker. + +The matched case findings are part of the pool and are adjudicated like any other. + +## What is reported + +Per case: detected / unstable / unanswered, fixed-side and benign false alerts, completion (questions completed / +total) for each of the six scans, stability, and seconds. Overall: + +- recall with a Wilson 95% interval; +- sampled precision with a Wilson 95% interval; +- total fixed-side and benign false alerts; +- completion over all scans; +- pooled stability. + +These are reported against the M4 profile: at least 95% precision, at least 90% recall, at least 95% completion, +and zero fixed or benign false alerts. A gate that is not met is reported as not met. + +## What may not happen + +- No rule, fact, query or scoring change informed by these results before the result is committed. +- Any change made after it is a retune. A retune can only be qualified on a NEW untouched population. +- No result of a case whose licence is marked "verify" is published until the licence is checked.