diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index b1f7eb2..16db072 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -79,7 +79,7 @@ ## Group 2 — Scale it (the phase that can reshape the rest) -- [ ] 2.1 A pinned known-vulnerable checkout +- [x] 2.1 A pinned known-vulnerable checkout - `benchmarks/repos/.toml`: url, commit, the known CVE, the file and function it lives in, licence. Start with one small enough to iterate on. Fetch is explicit and offline-by-default, like the pointer pairs. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index e9eea90..6d3e8d2 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,13 +1,13 @@ { "counts": { - "load_bearing": 88, + "load_bearing": 89, "standalone_capability": 7 }, - "generated": "2026-09-09 (regenerated after contributor-scan group 1 added regions and the repo driver)", + "generated": "2026-09-09 (regenerated after contributor-scan 2.1 added the pinned-checkout recipes)", "kept_with_a_lever_stripped": { "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." }, - "modules": 95, + "modules": 96, "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], "removed_in_this_pass": { diff --git a/benchmarks/repos/README.md b/benchmarks/repos/README.md new file mode 100644 index 0000000..17f0769 --- /dev/null +++ b/benchmarks/repos/README.md @@ -0,0 +1,63 @@ +# Pinned known-vulnerable checkouts + +The pair corpus scores excerpts: a vulnerable function and its fixed twin, fifteen lines each, handed to the +engine with the answer already narrowed down. That measures arbitration. It cannot measure anything about a +repository — region ranking, the budget, one CPG over thousands of files, or whether a verdict that holds on +an excerpt still holds when the source, the sink and the guard live in three different modules. + +These recipes name real checkouts so those questions can be measured instead of assumed. + +## What a recipe is + +One TOML file per repository, naming a url, a **full 40-character commit SHA**, the licence, and every known +vulnerability with the file, function and line it lives at. + +``` +ousast repos # what is pinned, and whether it is checked out +ousast repos --repo vampi --fetch # materialise one (network; CI never does this) +ousast repos --json # the same, for an artifact +``` + +Nothing here is vendored. The repositories stay in their own projects under their own licences; the recipe +records the licence so a measurement artifact can say what it ran against. + +## Three rules, and why + +**Offline by default.** A recipe is inert until someone asks for the network — `--fetch` for one run, or +`OPENULTRASAST_REPOS_NETWORK=1`. CI sets neither, and the test suite clears both. This is the same rule the +pointer pairs follow, for the same reason: a test suite that quietly reaches the network is a test suite that +fails for reasons that have nothing to do with the code. + +**Pinned, not tracked.** `commit` must be a full SHA; a branch or a tag is rejected at load. A tag gets +re-cut and a branch moves every week, and either would make the same recipe mean different code next month +while the baseline it produced still carried the old numbers. + +**A recipe declares what it can measure.** `measures` is `detection`, `envelope`, or both, and every known +vulnerability states `in_scope` explicitly — whether it belongs to a family this engine arbitrates at all. +There is no default, because the default would silently decide the most important question about the result: + +> An out-of-bounds read in a C library is a fine envelope test and an impossible detection test. Recording +> "not found" against it as a miss is the easiest way to make a measurement lie. + +## Verification + +`ousast repos` does not trust a recipe. Against a materialised checkout it checks that HEAD is the pinned +commit, that the licence file is really there, and that each declared function appears above its declared +line in its declared file. A recipe that contradicts the code exits non-zero; a recipe that simply has not +been fetched does not, or CI could never run this command. + +That check earned its place immediately: the first libpng recipe named `LICENSE.md`, which is the filename at +HEAD today and not at the pinned commit, where it is still `LICENSE`. + +## What is pinned + +| repo | language | size | measures | why | +|---|---|---|---|---| +| `vampi` | python | 520 lines, 11 files | detection | Small enough that a detection result is not confounded by budget or ranking. Its routes are registered through an OpenAPI `operationId` rather than decorators, so it exercises registration-based entry-point mapping. Excerpts of it are already in the pair corpus, so a whole-repository run says directly whether a verdict survives the move off the excerpt. | +| `libpng` | c | 91k lines, 101 files | envelope | Whether one CPG over a real C library is affordable at all: build time, query time, peak memory, and whether the ranker yields a sane number of regions. Its CVE is out of scope by construction. | + +## Baselines + +Task 6.1 turns a scan of a pinned checkout into a committed baseline, so a change to the fact tables or the +queries can be read as findings gained, findings lost and rung movements (Req 10.1–10.4). Until then these +recipes exist to be measured against by hand, and the first such measurement is task 2.2. diff --git a/benchmarks/repos/libpng.toml b/benchmarks/repos/libpng.toml new file mode 100644 index 0000000..fcfaf49 --- /dev/null +++ b/benchmarks/repos/libpng.toml @@ -0,0 +1,34 @@ +# libpng -- 91k lines of C across 101 files. +# +# This is the ENVELOPE checkout, and the only question it answers is whether one CPG over a real C library +# is affordable: build time, query time, peak memory, and whether the region ranker produces a sane number +# of regions rather than thousands. +# +# Its known CVE is deliberately marked out of scope. An out-of-bounds read is not a family this engine +# arbitrates -- there is no taint sanitizer lattice, guard dominance or constant abstraction that decides +# it -- so "not found" here is not a miss, and recording it as one would be the easiest way to make a +# measurement lie. + +name = "libpng" +url = "https://github.com/glennrp/libpng.git" +commit = "cf53e54ccf631d8de68a30ebb3b8eeed1d6f1a08" +language = "c" +license = "libpng-2.0" +license_file = "LICENSE" +measures = ["envelope"] +note = """ +The pinned commit is the parent of 788a624d7387a758ffd5c7ab010f1870dea753a1, the commit that added the +missing clamp; `git describe` calls it v1.6.51-1-gcf53e54cc. The licence file is `LICENSE` at this commit; it was renamed +to `LICENSE.md` later, which is why the recipe names the file and the verifier checks it. Nothing is vendored from this repository -- the +recipe fetches it on request and CI never does. +""" + +[[known]] +id = "CVE-2025-66293" +family = "memory_safety" +weakness = "out-of-bounds read: an unclamped composited component indexes the sRGB conversion table" +file = "pngread.c" +function = "png_image_read_composite" +line = 3279 +reference = "fixed in 788a624d7387a758ffd5c7ab010f1870dea753a1; the CVE is named in the code comment added by d1ec15266204deb4102d2b5e40216bb7637261c8" +in_scope = false diff --git a/benchmarks/repos/vampi.toml b/benchmarks/repos/vampi.toml new file mode 100644 index 0000000..5f792db --- /dev/null +++ b/benchmarks/repos/vampi.toml @@ -0,0 +1,47 @@ +# VAmPI -- a deliberately vulnerable Flask API, small enough to iterate on. +# +# This is the DETECTION checkout. The corpus already scores excerpts cut out of VAmPI, and one of them +# (users/update_password) was diagnosed as a miss. Scanning the whole repository asks the question the +# excerpts cannot: does a verdict survive when the route registration, the handler and the query live in +# three different modules, and when the region ranker has to find the handler rather than being handed it? +# +# Its routes are registered through an OpenAPI `operationId`, not a decorator, so it also exercises the +# registration-based entry-point mapping rather than the decorator path. + +name = "vampi" +url = "https://github.com/erev0s/VAmPI.git" +commit = "f16052dce83f05847133ec98f01c5193a41de7d8" +language = "python" +license = "MIT" +license_file = "LICENSE" +measures = ["detection"] +note = """ +520 lines across 11 Python files: too small to say anything about the envelope, and chosen for exactly that +reason -- a detection result here is not confounded by budget exhaustion or region ranking. +""" + +# VAmPI serves the vulnerable code paths behind a global switch, `int(os.getenv('vulnerable', 1))` in app.py. +# The default is already vulnerable; it is pinned here because a measurement that silently ran the patched +# branch would report a clean repository and look like a detection failure. +[environment] +vulnerable = "1" + +[[known]] +id = "VAmPI-SQLI" +family = "injection" +weakness = "SQL injection: the username is interpolated into the query string at line 72 and executed at 73" +file = "models/user_model.py" +function = "get_user" +line = 73 +reference = "README.md 'List of Vulnerabilities'; the vulnerable branch is guarded by `if vuln:`" +in_scope = true + +[[known]] +id = "VAmPI-BOLA" +family = "access_control" +weakness = "unauthorized password change: the path username is used instead of the authenticated token subject" +file = "api_views/users.py" +function = "update_password" +line = 187 +reference = "README.md 'Unauthorized Password Change'; the non-vulnerable branch uses resp['sub']" +in_scope = true diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index f3e030e..9adb652 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -60,6 +60,16 @@ from .provenance import fingerprint from .rank import rank_obligations, rank_targets, write_rankings from .regress import TRIGGERABLE, CandidateVerdict, run_regression, write_verdicts from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report +from .repos import ( + DEFAULT_REPO_DIR, + RepoUnavailable, + checkout_path, + load_repo_recipes, + recipe_payload, +) +from .repos import resolve as resolve_repo +from .repos import select as select_repo +from .repos import verify as verify_repo from .ruleset import DEFAULT_RULESET_DIR, load_ruleset from .run import ScanRun, create_scan_run from .sandbox import resolve_sandbox_probe, resolve_sandbox_runner @@ -159,6 +169,15 @@ def main(argv: list[str] | None = None) -> int: pairs.add_argument("--pointers", action="store_true", help="allow network for non-vendored pointer pairs this run (nightly; CI never)") pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON") + repos = subparsers.add_parser( + "repos", + help="pinned known-vulnerable checkouts: list them, and verify a recipe against the code it names", + ) + repos.add_argument("--dir", type=Path, default=DEFAULT_REPO_DIR) + repos.add_argument("--repo", default="all", help="one recipe by name, or all") + repos.add_argument("--fetch", action="store_true", help="allow network to materialise missing checkouts (CI never)") + repos.add_argument("--json", action="store_true") + subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration") model_cmd = subparsers.add_parser("model", help="maintainer: the model layer that arbitrates detector claims") model_sub = model_cmd.add_subparsers(dest="model_command", required=True) @@ -199,6 +218,8 @@ def main(argv: list[str] | None = None) -> int: split=args.split, pointers=args.pointers, ) + if args.command == "repos": + return _repos(args.dir, args.repo, fetch=args.fetch, json_out=args.json) if args.command == "model": return _model_candidates(args) if args.command == "mcp": @@ -803,6 +824,45 @@ def _pairs( return 0 +def _repos(directory: Path, name: str, *, fetch: bool, json_out: bool) -> int: + """List the pinned checkouts and check each recipe against the code it claims to describe. + + Offline by default: without ``--fetch`` a missing checkout is reported as missing, not fetched. Exit is + non-zero only when a materialised checkout CONTRADICTS its recipe -- a recipe that merely has not been + fetched is not a failure, or CI could never run this. + """ + recipes = select_repo(load_repo_recipes(directory), name) + if not recipes: + raise SystemExit(f"no repository recipe matches {name!r} in {directory}") + + rows: list[dict[str, object]] = [] + contradicted = 0 + for recipe in recipes: + problems: tuple[str, ...] = () + try: + root = resolve_repo(recipe, fetch=True if fetch else None) + except RepoUnavailable: + state, path = "absent", checkout_path(recipe) + else: + problems = verify_repo(recipe, root) + state, path = ("contradicted" if problems else "verified"), root + contradicted += 1 if problems else 0 + rows.append({**recipe_payload(recipe), "state": state, "path": str(path), "problems": list(problems)}) + + if not json_out: + in_scope = len(recipe.in_scope) + measures = ",".join(recipe.measures) + print(f"{recipe.name:10} {state:13} {recipe.language:8} measures={measures} known={len(recipe.known)} in_scope={in_scope}") + for problem in problems: + print(f" ! {problem}") + if state == "absent": + print(f" fetch with: ousast repos --repo {recipe.name} --fetch") + + if json_out: + print(json.dumps(rows, indent=2, sort_keys=True)) + return 1 if contradicted else 0 + + def _model_skipped(reason: str) -> dict[str, object]: """A skipped stage reports the same SHAPE as a run one, plus why it skipped. diff --git a/src/openultrasast/repos.py b/src/openultrasast/repos.py new file mode 100644 index 0000000..e0674ef --- /dev/null +++ b/src/openultrasast/repos.py @@ -0,0 +1,285 @@ +"""Pinned known-vulnerable repository checkouts (Req 4.1). + +The corpus scores excerpts. Nothing in it exercises a whole repository: region ranking, the budget, one CPG +over thousands of files, or whether a verdict that holds on a fifteen-line excerpt still holds when the +source, the sink and the guard live in three different modules. These recipes name real checkouts so that +question can be measured instead of assumed. + +Two rules carried over from the pointer pairs, for the same reasons: + +* **Offline by default.** A recipe is inert until someone asks for the network, by ``--fetch`` for one run or + ``OPENULTRASAST_REPOS_NETWORK=1``. CI sets neither, so CI never fetches. +* **Pinned, not tracked.** ``commit`` must be a full SHA. A branch or a tag would make the same recipe mean + different code next month, and a measurement against moving code is not a baseline. + +A third rule is specific to these: **a recipe declares what it can measure.** ``in_scope`` says, per known +vulnerability, whether it belongs to a family the model layer arbitrates at all. A memory-safety CVE in a C +library is a fine envelope test and an impossible detection test, and a measurement that does not separate +the two invites reading "not found" as a miss when it was never in scope. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import tomllib +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +REPO_CACHE_ENV = "OPENULTRASAST_REPO_CACHE" +REPOS_NETWORK_ENV = "OPENULTRASAST_REPOS_NETWORK" +DEFAULT_REPO_CACHE = Path.home() / ".cache" / "openultrasast" / "repos" +DEFAULT_REPO_DIR = Path("benchmarks/repos") + +MEASURES = ("envelope", "detection") +_SHA = re.compile(r"^[0-9a-f]{40}$") +_RECIPE_REQUIRED = ("name", "url", "commit", "language", "license", "license_file", "measures") +_KNOWN_REQUIRED = ("id", "family", "weakness", "file", "function", "line", "reference") +# How far above the declared line the function signature may sit. This is a consistency check between +# three fields, not a parse: C functions in libpng run past ninety lines, so a tight window rejects +# honest recipes, and a window this size still catches a line number that belongs to a different +# function entirely. +_FUNCTION_WINDOW = 200 + + +class RepoRecipeError(ValueError): + """A recipe that cannot be trusted to name one checkout.""" + + +class RepoUnavailable(RuntimeError): + """The checkout is not on disk and the network was not offered.""" + + +@dataclass(frozen=True) +class KnownVulnerability: + """One documented vulnerability, and whether this engine could decide it even in principle.""" + + id: str + family: str + weakness: str + file: str + function: str + line: int + reference: str + in_scope: bool + + @property + def location(self) -> str: + return f"{self.file}:{self.function}:{self.line}" + + +@dataclass(frozen=True) +class RepoRecipe: + name: str + url: str + commit: str + language: str + license: str + license_file: str + measures: tuple[str, ...] + known: tuple[KnownVulnerability, ...] + environment: Mapping[str, str] + note: str + + @property + def in_scope(self) -> tuple[KnownVulnerability, ...]: + """The known vulnerabilities a rung can be reported for; the rest are envelope-only.""" + return tuple(item for item in self.known if item.in_scope) + + +def load_repo_recipes(directory: Path = DEFAULT_REPO_DIR) -> tuple[RepoRecipe, ...]: + if not directory.is_dir(): + return () + return tuple(load_repo_recipe(path) for path in sorted(directory.glob("*.toml"))) + + +def load_repo_recipe(path: Path) -> RepoRecipe: + data = tomllib.loads(path.read_text()) + missing = [key for key in _RECIPE_REQUIRED if not data.get(key)] + if missing: + raise RepoRecipeError(f"{path.name}: missing {', '.join(missing)}") + + commit = str(data["commit"]) + if not _SHA.match(commit): + # A tag moves when it is re-cut and a branch moves every week. Either would make this recipe name + # different code over time while the baseline it produced kept the old numbers. + raise RepoRecipeError(f"{path.name}: commit must be a full 40-character SHA, not {commit!r}") + + measures = tuple(str(item) for item in data["measures"]) + unknown = [item for item in measures if item not in MEASURES] + if unknown: + raise RepoRecipeError(f"{path.name}: unknown measures {unknown}; expected any of {list(MEASURES)}") + + known = tuple(_known(item, path.name) for item in data.get("known", ())) + if "detection" in measures and not any(item.in_scope for item in known): + raise RepoRecipeError(f"{path.name}: declares detection but names no in-scope vulnerability to detect") + + return RepoRecipe( + name=str(data["name"]), + url=str(data["url"]), + commit=commit, + language=str(data["language"]), + license=str(data["license"]), + license_file=str(data["license_file"]), + measures=measures, + known=known, + environment={str(k): str(v) for k, v in dict(data.get("environment", {})).items()}, + note=str(data.get("note", "")), + ) + + +def _known(item: object, name: str) -> KnownVulnerability: + if not isinstance(item, dict): + raise RepoRecipeError(f"{name}: each [[known]] entry must be a table") + missing = [key for key in _KNOWN_REQUIRED if item.get(key) in (None, "")] + if missing: + raise RepoRecipeError(f"{name}: [[known]] missing {', '.join(missing)}") + if "in_scope" not in item: + # Deliberately not defaulted. Whether the engine could ever decide this is the difference between a + # miss and a non-question, and a default would silently pick one. + raise RepoRecipeError(f"{name}: [[known]] {item['id']} must state in_scope explicitly") + return KnownVulnerability( + id=str(item["id"]), + family=str(item["family"]), + weakness=str(item["weakness"]), + file=str(item["file"]), + function=str(item["function"]), + line=int(item["line"]), + reference=str(item["reference"]), + in_scope=bool(item["in_scope"]), + ) + + +def repo_cache_dir() -> Path: + """Checkout root; outside the repository unless the operator points it elsewhere.""" + return Path(os.environ.get(REPO_CACHE_ENV) or DEFAULT_REPO_CACHE).expanduser() + + +def network_allowed(fetch: bool | None = None) -> bool: + """``repos --fetch`` for one run, or OPENULTRASAST_REPOS_NETWORK=1; CI sets neither.""" + if fetch is not None: + return fetch + return os.environ.get(REPOS_NETWORK_ENV, "") == "1" + + +def checkout_path(recipe: RepoRecipe, cache_root: Path | None = None) -> Path: + """Where this exact commit lives. Keyed by SHA, so a re-pin never overwrites an older baseline's source.""" + return (cache_root or repo_cache_dir()) / recipe.name / recipe.commit[:12] + + +def resolve(recipe: RepoRecipe, *, fetch: bool | None = None, cache_root: Path | None = None) -> Path: + """The checkout on disk, fetching only if the network was offered. + + A warm cache spares the fetch; it never turns an offline run into a networked one. + """ + dest = checkout_path(recipe, cache_root) + if (dest / ".git").is_dir(): + return dest + if not network_allowed(fetch): + raise RepoUnavailable( + f"{recipe.name} is not checked out at {recipe.commit[:12]}; run with --fetch or {REPOS_NETWORK_ENV}=1 to fetch it" + ) + _fetch(recipe, dest) + return dest + + +def _fetch(recipe: RepoRecipe, dest: Path) -> None: + """Fetch just the pinned commit. Falls back to a full clone where the server refuses a SHA fetch.""" + dest.mkdir(parents=True, exist_ok=True) + try: + _git(dest, "init", "--quiet") + _git(dest, "remote", "add", "origin", recipe.url) + _git(dest, "fetch", "--quiet", "--depth", "1", "origin", recipe.commit) + _git(dest, "checkout", "--quiet", "FETCH_HEAD") + except subprocess.CalledProcessError: + for child in dest.iterdir(): + _remove(child) + _git(dest.parent, "clone", "--quiet", recipe.url, dest.name) + _git(dest, "checkout", "--quiet", recipe.commit) + + +def _remove(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + for child in path.iterdir(): + _remove(child) + path.rmdir() + else: + path.unlink() + + +def _git(cwd: Path, *args: str) -> str: + result = subprocess.run( # noqa: S603 - fixed program, arguments from a committed recipe + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + timeout=1800, + ) + return result.stdout.strip() + + +def verify(recipe: RepoRecipe, root: Path) -> tuple[str, ...]: + """Problems with a materialised checkout: empty means the recipe describes what is actually there. + + This is what keeps a recipe from drifting into fiction. Every field that claims something about the code + -- the commit, the licence file, and each vulnerability's file and function -- is checked against the + checkout rather than trusted. + """ + problems: list[str] = [] + try: + head = _git(root, "rev-parse", "HEAD") + except (subprocess.CalledProcessError, OSError) as exc: + return (f"cannot read HEAD: {exc}",) + if head != recipe.commit: + problems.append(f"HEAD is {head[:12]}, recipe pins {recipe.commit[:12]}") + if not (root / recipe.license_file).is_file(): + problems.append(f"license_file {recipe.license_file} is not in the checkout") + for item in recipe.known: + problems.extend(_verify_known(item, root)) + return tuple(problems) + + +def _verify_known(item: KnownVulnerability, root: Path) -> tuple[str, ...]: + path = root / item.file + if not path.is_file(): + return (f"{item.id}: {item.file} is not in the checkout",) + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + if not 1 <= item.line <= len(lines): + return (f"{item.id}: line {item.line} is outside {item.file} ({len(lines)} lines)",) + # The declared line is where the vulnerability is, not where the function opens, so look for the name in + # a window above it rather than on it. + window = lines[max(item.line - _FUNCTION_WINDOW, 0) : item.line] + if not any(item.function in line for line in window): + return (f"{item.id}: {item.function} is not within {_FUNCTION_WINDOW} lines above {item.file}:{item.line}",) + return () + + +def recipe_payload(recipe: RepoRecipe) -> dict[str, object]: + """The recipe as it should appear in a measurement artifact (Req 4.2, 10.4).""" + return { + "name": recipe.name, + "url": recipe.url, + "commit": recipe.commit, + "language": recipe.language, + "license": recipe.license, + "measures": list(recipe.measures), + "known": [ + { + "id": item.id, + "family": item.family, + "weakness": item.weakness, + "location": item.location, + "in_scope": item.in_scope, + } + for item in recipe.known + ], + } + + +def select(recipes: Sequence[RepoRecipe], name: str | None) -> tuple[RepoRecipe, ...]: + if name in (None, "all"): + return tuple(recipes) + return tuple(recipe for recipe in recipes if recipe.name == name) diff --git a/tests/conftest.py b/tests/conftest.py index 6b1aab4..4d65793 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,17 @@ def _pointer_pairs_offline(monkeypatch: pytest.MonkeyPatch, tmp_path_factory: py monkeypatch.setenv("OPENULTRASAST_PAIR_CACHE", str(tmp_path_factory.mktemp("pair-cache"))) +@pytest.fixture(autouse=True) +def _repos_offline(monkeypatch: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory) -> None: + """Req 4.1: pinned repository recipes never fetch during a test run, whatever the operator's shell exports. + + The cache is redirected as well as the switch cleared, so a developer with warm checkouts and a test that + forgets to pass an explicit root still cannot read them by accident. + """ + monkeypatch.delenv("OPENULTRASAST_REPOS_NETWORK", raising=False) + monkeypatch.setenv("OPENULTRASAST_REPO_CACHE", str(tmp_path_factory.mktemp("repo-cache"))) + + @pytest.fixture def assert_cold_of_harnessx(): # type: ignore[no-untyped-def] """Run ``code`` in a fresh interpreter and assert it imported no ``harnessx`` module. diff --git a/tests/test_repos.py b/tests/test_repos.py new file mode 100644 index 0000000..9ed3330 --- /dev/null +++ b/tests/test_repos.py @@ -0,0 +1,223 @@ +"""Pinned known-vulnerable checkouts (Req 4.1). + +Every test here is offline. The one that fetches builds a git repository in ``tmp_path`` and uses it as the +recipe's url, which exercises the real fetch path -- ``git init``, ``fetch``, ``checkout`` -- with no network. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from openultrasast.repos import ( + DEFAULT_REPO_DIR, + RepoRecipeError, + RepoUnavailable, + checkout_path, + load_repo_recipe, + load_repo_recipes, + network_allowed, + recipe_payload, + resolve, + select, + verify, +) + +RECIPE = """ +name = "sample" +url = "{url}" +commit = "{commit}" +language = "python" +license = "MIT" +license_file = "LICENSE" +measures = ["detection"] + +[[known]] +id = "SAMPLE-1" +family = "injection" +weakness = "the parameter is interpolated into the query" +file = "app.py" +function = "handler" +line = 3 +reference = "README" +in_scope = true +""" + + +def _git(cwd: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() + + +@pytest.fixture +def origin(tmp_path: Path) -> tuple[Path, str]: + """A real git repository standing in for a remote, and the SHA a recipe would pin.""" + root = tmp_path / "origin" + (root / "sub").mkdir(parents=True) + (root / "LICENSE").write_text("MIT") + (root / "app.py").write_text("def handler(name):\n q = f'select {name}'\n execute(q)\n") + _git(root, "init", "--quiet", "-b", "main") + _git(root, "config", "user.email", "t@example.com") + _git(root, "config", "user.name", "t") + _git(root, "add", "-A") + _git(root, "commit", "--quiet", "-m", "sample") + return root, _git(root, "rev-parse", "HEAD") + + +@pytest.fixture +def recipe_path(tmp_path: Path, origin: tuple[Path, str]) -> Path: + root, commit = origin + path = tmp_path / "sample.toml" + path.write_text(RECIPE.format(url=root.as_uri(), commit=commit)) + return path + + +def test_recipe_round_trips(recipe_path: Path) -> None: + recipe = load_repo_recipe(recipe_path) + assert recipe.name == "sample" + assert recipe.measures == ("detection",) + assert [item.id for item in recipe.in_scope] == ["SAMPLE-1"] + assert recipe_payload(recipe)["known"] == [ + { + "id": "SAMPLE-1", + "family": "injection", + "weakness": "the parameter is interpolated into the query", + "location": "app.py:handler:3", + "in_scope": True, + } + ] + + +def test_a_branch_is_not_a_pin(tmp_path: Path) -> None: + """A tag or branch would make the same recipe mean different code next month.""" + path = tmp_path / "loose.toml" + path.write_text(RECIPE.format(url="https://example.invalid/x.git", commit="main")) + with pytest.raises(RepoRecipeError, match="full 40-character SHA"): + load_repo_recipe(path) + + +def test_in_scope_must_be_stated(tmp_path: Path, origin: tuple[Path, str]) -> None: + """Defaulting it would silently decide whether 'not found' is a miss or a non-question.""" + _, commit = origin + path = tmp_path / "quiet.toml" + path.write_text(RECIPE.format(url="https://example.invalid/x.git", commit=commit).replace("in_scope = true\n", "")) + with pytest.raises(RepoRecipeError, match="must state in_scope explicitly"): + load_repo_recipe(path) + + +def test_detection_needs_something_in_scope(tmp_path: Path, origin: tuple[Path, str]) -> None: + _, commit = origin + path = tmp_path / "empty.toml" + path.write_text(RECIPE.format(url="https://example.invalid/x.git", commit=commit).replace("in_scope = true", "in_scope = false")) + with pytest.raises(RepoRecipeError, match="names no in-scope vulnerability"): + load_repo_recipe(path) + + +def test_unknown_measure_is_rejected(tmp_path: Path, origin: tuple[Path, str]) -> None: + _, commit = origin + path = tmp_path / "odd.toml" + path.write_text(RECIPE.format(url="https://example.invalid/x.git", commit=commit).replace('["detection"]', '["vibes"]')) + with pytest.raises(RepoRecipeError, match="unknown measures"): + load_repo_recipe(path) + + +def test_offline_by_default(recipe_path: Path, tmp_path: Path) -> None: + """No switch, no fetch -- this is what keeps CI from reaching the network.""" + assert network_allowed() is False + with pytest.raises(RepoUnavailable, match="--fetch"): + resolve(load_repo_recipe(recipe_path), cache_root=tmp_path / "cache") + + +def test_the_switch_is_read_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENULTRASAST_REPOS_NETWORK", "1") + assert network_allowed() is True + assert network_allowed(False) is False, "an explicit flag beats the environment" + + +def test_resolve_fetches_then_serves_from_cache(recipe_path: Path, tmp_path: Path) -> None: + recipe = load_repo_recipe(recipe_path) + cache = tmp_path / "cache" + + root = resolve(recipe, fetch=True, cache_root=cache) + + assert root == checkout_path(recipe, cache) + assert (root / "app.py").is_file() + assert verify(recipe, root) == (), "the recipe describes what was actually fetched" + # A warm cache spares the fetch; it never turns an offline run into a networked one. + assert resolve(recipe, cache_root=cache) == root + + +def test_checkout_is_keyed_by_commit(recipe_path: Path, tmp_path: Path) -> None: + """Re-pinning must not overwrite the source an older baseline was measured against.""" + recipe = load_repo_recipe(recipe_path) + other = checkout_path(recipe, tmp_path) + assert other.name == recipe.commit[:12] + assert other.parent.name == "sample" + + +def test_verify_catches_the_wrong_commit(recipe_path: Path, tmp_path: Path, origin: tuple[Path, str]) -> None: + root_repo, _ = origin + recipe = load_repo_recipe(recipe_path) + root = resolve(recipe, fetch=True, cache_root=tmp_path / "cache") + (root_repo / "app.py").write_text("def handler(name):\n pass\n") + _git(root_repo, "commit", "--quiet", "-am", "moved on") + moved = _git(root_repo, "rev-parse", "HEAD") + _git(root, "fetch", "--quiet", "origin", moved) + _git(root, "checkout", "--quiet", "FETCH_HEAD") + + assert any("HEAD is" in problem for problem in verify(recipe, root)) + + +def test_verify_catches_a_recipe_that_describes_absent_code(recipe_path: Path, tmp_path: Path) -> None: + recipe = load_repo_recipe(recipe_path) + root = resolve(recipe, fetch=True, cache_root=tmp_path / "cache") + (root / "app.py").unlink() + (root / "LICENSE").unlink() + + problems = verify(recipe, root) + + assert any("LICENSE" in problem for problem in problems) + assert any("app.py is not in the checkout" in problem for problem in problems) + + +def test_verify_catches_a_line_outside_the_file(recipe_path: Path, tmp_path: Path) -> None: + recipe = load_repo_recipe(recipe_path) + root = resolve(recipe, fetch=True, cache_root=tmp_path / "cache") + (root / "app.py").write_text("only one line\n") + + assert any("outside" in problem for problem in verify(recipe, root)) + + +def test_verify_catches_a_function_that_is_not_there(recipe_path: Path, tmp_path: Path) -> None: + recipe = load_repo_recipe(recipe_path) + root = resolve(recipe, fetch=True, cache_root=tmp_path / "cache") + (root / "app.py").write_text("a\nb\nc\n") + + assert any("handler is not within" in problem for problem in verify(recipe, root)) + + +def test_select_by_name(recipe_path: Path) -> None: + recipes = (load_repo_recipe(recipe_path),) + assert select(recipes, "all") == recipes + assert select(recipes, "sample") == recipes + assert select(recipes, "absent") == () + + +def test_the_committed_recipes_parse() -> None: + """The corpus itself, offline: every field present, every commit pinned, every scope decided.""" + recipes = load_repo_recipes(DEFAULT_REPO_DIR) + + assert {recipe.name for recipe in recipes} >= {"vampi", "libpng"} + for recipe in recipes: + assert recipe.license, f"{recipe.name} names no licence" + assert recipe.known, f"{recipe.name} names no known vulnerability" + assert all(item.line > 0 for item in recipe.known) + + +def test_the_envelope_checkout_claims_no_detection() -> None: + """libpng's CVE is a memory-safety bug: recording 'not found' as a miss would make the number lie.""" + libpng = next(recipe for recipe in load_repo_recipes(DEFAULT_REPO_DIR) if recipe.name == "libpng") + + assert "detection" not in libpng.measures + assert libpng.in_scope == ()