mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
d1dc0d1131
Every reserved case verified with git and file reads only -- nothing scanned: pins present and ordered, the sink present at the vulnerable pin, the fix touching the sink (or LearnPress's declared source-side fix site), and the sink file hashed at both pins (benchmarks/independent/freeze.py, record freeze-v1.json). Each case now carries a benign control from its own history, reviewed by diff. The mechanical keyword filter was not enough: it proposed an access restriction, a filter check, API-token scopes, a login change and a TLS-verification option, all rejected; Ultimate Member and alerta got hand-picked controls (a WPCS reformat; a dependency change with both sides still vulnerable). The guard test requires a benign pair per case once frozen. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
70 lines
3.4 KiB
Python
70 lines
3.4 KiB
Python
"""The independent population is reserved, well formed, and untouched by anything in the development tree.
|
|
|
|
Every rule this project has learned was learned on its development subjects, so those can no longer measure
|
|
whether the rules generalise. This reservation is the population that can -- until something scans it. The
|
|
check below fails the moment a reserved repository is referenced by a recipe, catalog, manifest or measurement
|
|
anywhere else, which is how an evaluation set quietly becomes a development one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import tomllib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
POPULATION = ROOT / "benchmarks" / "independent" / "population-v1.toml"
|
|
FAMILIES = {"injection", "untrusted_destination", "config_secrets"}
|
|
SHA = re.compile(r"[0-9a-f]{40}")
|
|
|
|
|
|
def _population() -> dict:
|
|
return tomllib.loads(POPULATION.read_text())
|
|
|
|
|
|
def test_the_reservation_is_well_formed() -> None:
|
|
data = _population()
|
|
assert data["status"] in {"reserved-unscanned", "frozen"}
|
|
cases = data["case"]
|
|
ids = [case["id"] for case in cases]
|
|
assert len(ids) == len(set(ids)) >= 10
|
|
for case in cases:
|
|
assert case["family"] in FAMILIES, case["id"]
|
|
assert SHA.fullmatch(case["vulnerable"]) and SHA.fullmatch(case["fixed"]), case["id"]
|
|
assert case["vulnerable"] != case["fixed"], case["id"]
|
|
assert case["repo"].startswith("https://github.com/"), case["id"]
|
|
assert case["advisory"] and case["sink"] and case["privilege"], case["id"]
|
|
if data["status"] == "frozen":
|
|
# Frozen means every case carries its benign control: a change from the same history, reviewed as not a
|
|
# security change, whose two sides a correct analyzer must not tell apart.
|
|
for case in cases:
|
|
benign = case.get("benign") or {}
|
|
assert SHA.fullmatch(benign.get("base", "")) and SHA.fullmatch(benign.get("tip", "")), case["id"]
|
|
assert (POPULATION.parent / data["freeze_record"]).is_file()
|
|
languages = {case["language"] for case in cases}
|
|
assert {"php", "python"} <= languages and languages & {"javascript", "typescript"}
|
|
|
|
|
|
def test_a_multi_language_repository_is_reserved() -> None:
|
|
"""The declared v0.1 target is a WordPress plugin: PHP plus its admin JavaScript (task 13.4)."""
|
|
data = _population()
|
|
by_id = {case["id"]: case for case in data["case"]}
|
|
members = data["multi_language_members"]
|
|
assert members and all(by_id[m]["language"] == "php" and "javascript" in by_id[m].get("also", []) for m in members)
|
|
|
|
|
|
def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None:
|
|
reserved = {case["repo"].rstrip("/").lower() for case in _population()["case"]}
|
|
names = {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved}
|
|
offenders = []
|
|
for top in ("benchmarks", "src", "tests"):
|
|
for path in (ROOT / top).rglob("*"):
|
|
if not path.is_file() or POPULATION.parent in path.parents or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}:
|
|
continue
|
|
try:
|
|
text = path.read_text(errors="ignore").lower()
|
|
except OSError:
|
|
continue
|
|
offenders += [f"{path.relative_to(ROOT)}: {name}" for name in names if f"github.com/{name}" in text]
|
|
assert not offenders, "a reserved repository is referenced outside the reservation: " + "; ".join(offenders[:10])
|