diff --git a/.agents/skills/openultrasast-scan/SKILL.md b/.agents/skills/openultrasast-scan/SKILL.md index c69e2ff..11ede43 100644 --- a/.agents/skills/openultrasast-scan/SKILL.md +++ b/.agents/skills/openultrasast-scan/SKILL.md @@ -38,7 +38,7 @@ uv run ousast mcp # the MCP server over stdio, for agent hosts | --- | --- | --- | --- | | `quick` | language-scoped pattern rules, entry-point hints, ranking, verification, score; deterministic | nothing | - | | `standard` | the MAP stage: complexity map, semantic overlay, authorization obligations, the Joern model layer (one code property graph per repository; taint, guard-dominance and configuration questions), deterministic fusion | Joern (shipped in the Docker image, with `php-cli` for PHP); a provider key for the residual LLM question | `cpg_unavailable`; without a key the LLM question is not asked and only the graph's entailed findings are reported | -| `deep` | REGRESS: promoted candidates run as snippets in a Docker sandbox (no network, read-only source, non-root) and get a `triggerable` / `not_triggerable` / ... verdict in `verdicts.json` | a working `docker` | `sandbox_unavailable` | +| `deep` | nothing today: REGRESS was retired on 2026-10-13 (degradation `regress_retired`); execution proof belongs to the search-with-proof arm | - | - | Model stages read `DEEPSEEK_API_KEY` (every LLM call); with only `OPENROUTER_API_KEY` set, chat calls fall back to OpenRouter. An exported shell variable wins over `.env` silently. `[models] @@ -60,7 +60,7 @@ A degradation never changes the exit code. Read `manifest.json` for it. Each entry in `manifest.json` `degradations` is `{"stage": ..., "reason": ...}`. The ones to expect: - `cpg_unavailable` (stage `model`): no Joern on the host; the model layer ran nothing. -- `sandbox_unavailable` (stage `regress`): no working `docker`; deep mode fell back to standard. +- `regress_retired` (stage `regress`): deep mode requested the retired snippet stage; the scan equals standard. - `learning_endpoint_unavailable` (stage `model`): no provider key resolved. The graph's entailed findings are still reported; the residual question is skipped. This is recorded in `manifest.json` but has no line in `report.md`'s "What could not be analysed" section. diff --git a/.kiro/steering/overview.md b/.kiro/steering/overview.md index 6804aee..33975fb 100644 --- a/.kiro/steering/overview.md +++ b/.kiro/steering/overview.md @@ -259,4 +259,4 @@ Counted 2026-10-02 from each spec's `tasks.md` checkboxes and `spec.json`. > and evolutionary proposals come from the plane memory. The HarnessX specs in the table above are history. 151 source modules, 146 load-bearing and 5 standalone capabilities, 0 orphaned -(`benchmarks/measurements/2026-09-08-module-audit.json`, regenerated 2026-10-02); 2,023 tests collected. +(`benchmarks/measurements/2026-09-08-module-audit.json`, regenerated 2026-10-02); 2,023 tests collected (audit tool removed 2026-10-13). diff --git a/README.md b/README.md index a8d9cf6..57db4ad 100644 --- a/README.md +++ b/README.md @@ -119,12 +119,7 @@ ousast-with-judge scan . --mode standard # network on, the model is a `docker compose build` and `TARGET=/path/to/repo docker compose run --rm ousast scan /target` do the same without the wrapper. -`deep` adds REGRESS. REGRESS runs each reproduction snippet with `docker run` from wherever the -scan runs. The engine container has no Docker socket. So run `deep` in one of two ways: - -- from a host install with Joern on the PATH (`pyinfra @local ops/joern.py`, pinned and checksum- - verified) and a working `docker`; -- or accept the recorded `sandbox_unavailable` degradation. +`deep` requests the REGRESS stage, which was retired on 2026-10-13: the scan records a `regress_retired` degradation and otherwise equals `standard`. Execution proof now comes from the search-with-proof arm. Details, including the host install, are in [ops/README.md](ops/README.md). @@ -179,10 +174,9 @@ There are three modes. Each adds to the one before. taint, guard-dominance and configuration questions on that graph. A model is asked only the residual question. Without a key, that question is skipped and only the graph's entailed findings are reported. -- `deep` adds REGRESS. Promoted candidates are loaded by a small snippet in a Docker sandbox. Each - gets a `triggerable` or `not_triggerable` verdict. +- `deep` requests the REGRESS stage, which was retired on 2026-10-13: the scan records a `regress_retired` degradation and otherwise equals `standard`. Execution proof now comes from the search-with-proof arm. -A missing engine is a recorded degradation (`cpg_unavailable`, `sandbox_unavailable`), never a +A missing engine is a recorded degradation (`cpg_unavailable`, `regress_retired`), never a clean result. `--fail-on never|findings|verified|worth-fixing` sets the exit code. `--config openultrasast.toml` diff --git a/benchmarks/push/ontology_gap.py b/benchmarks/push/ontology_gap.py deleted file mode 100644 index c3ee3d6..0000000 --- a/benchmarks/push/ontology_gap.py +++ /dev/null @@ -1,147 +0,0 @@ -"""What a repository calls that the ontology does not model (contributor-scan 7.1). - -Five of seven families established nothing in the 2026-09 family census, and every one of them failed for an -ontology reason rather than an analysis one: the flow engine was ready and had nothing to aim at. Each gap -was found by investigating one family at a time, over days. This asks the question once. - -What it produces is a CANDIDATE LIST. A name here becomes a fact only when someone resolves it to a library -API, states the weakness class and records provenance -- the ontology describes libraries, frameworks and -language constructs, never repositories, and adding a name because a subject called it is how a -general-purpose tool becomes a fixture of its own benchmark. -""" - -from __future__ import annotations - -import argparse -import collections -import json -import time -from pathlib import Path -from typing import Any - -from openultrasast.cpg.backend import JoernBackend -from openultrasast.model.contracts import ExecutionBudget -from openultrasast.model.layout import is_test_path -from openultrasast.model.specs import config_specs, dominance_specs, taint_specs -from openultrasast.preprocess import enumerate_source_files - -# Build tooling a repository runs but does not ship. Named here rather than guessed from a path, because a -# file called Gruntfile is a build script wherever it sits. -_BUILD_SCRIPTS = frozenset({"Gruntfile.js", "gulpfile.js", "webpack.config.js", "rollup.config.js", "karma.conf.js"}) - - -def modelled_tokens(language: str) -> dict[str, tuple[str, ...]]: - """Every token any fact table names for this language, by the role it plays. - - Read from the specs rather than from the TOML, so the instrument measures what the analyser actually - consults: a fact the loader drops is not part of the ontology however neatly it is written. - """ - roles: dict[str, set[str]] = collections.defaultdict(set) - for family, spec in taint_specs(language=language).items(): - roles[f"sink:{family}"].update(spec.sinks) - roles["source"].update(spec.sources) - roles["sanitizer"].update(spec.sanitizers) - roles["sanitizer"].update(spec.safe_shape_sinks) - for spec in dominance_specs(language=language).values(): - roles["operation"].update(spec.operations) - roles["discharger"].update(spec.dischargers) - for spec in config_specs(language=language).values(): - roles["setting"].update(spec.settings) - return {role: tuple(sorted(tokens)) for role, tokens in roles.items()} - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--out", type=Path, required=True) - parser.add_argument("--root", type=Path, required=True) - parser.add_argument("--name", required=True) - parser.add_argument("--language", required=True) - parser.add_argument("--deadline", type=float, default=1800.0) - parser.add_argument("--top", type=int, default=60, help="how many unmodelled candidates to record") - args = parser.parse_args() - args.out.mkdir(parents=True, exist_ok=True) - - roles = modelled_tokens(args.language) - every = sorted({token for tokens in roles.values() for token in tokens}) - record: dict[str, Any] = { - "schema_version": 1, - "experiment": "ontology-coverage-v1", - "subject": args.name, - "language": args.language, - "root": str(args.root), - "claim_scope": ( - "A candidate list, not an ontology. A name here is a lead to a library API; it becomes a fact " - "only with a weakness class and provenance." - ), - "ontology_size": {role: len(tokens) for role, tokens in sorted(roles.items())}, - } - target = args.out / f"{args.name}-{args.language}.json" - - def save() -> None: - target.write_text(json.dumps(record, indent=2, default=str) + "\n") - - files = enumerate_source_files(args.root) - record["inputs"] = {"files": len(files), "bytes": sum(p.stat().st_size for p in files if p.is_file())} - save() - if not files: - record["error"] = "no readable source" - save() - return 2 - - started = time.monotonic() - budget = ExecutionBudget(started + args.deadline, 5.0) - backend = JoernBackend(execution_budget=budget) - graph = backend.build(args.root, language=args.language, execution_budget=budget) - if graph is None: - record["error"] = f"graph build failed: {backend.last_failure}" - save() - return 3 - - rows = backend.query( - graph.cpg_path, - "vocabulary", - {"modelled": ",".join(every), "sources": ",".join(roles.get("source", ()))}, - ) - record["seconds"] = round(time.monotonic() - started, 1) - if not isinstance(rows, list): - record["error"] = "the vocabulary query did not answer" - save() - return 4 - - known = [r for r in rows if isinstance(r, dict) and r.get("modelled")] - unknown = [r for r in rows if isinstance(r, dict) and not r.get("modelled")] - calls_total = sum(int(r.get("count", 0)) for r in rows if isinstance(r, dict)) - record["coverage"] = { - "distinct_call_names": len(rows), - "calls_total": calls_total, - "modelled_names": len(known), - "modelled_calls": sum(int(r.get("count", 0)) for r in known), - # The share of CALLS, not of names: one name called four hundred times matters more than four - # hundred called once, and a name count would flatter the ontology. - "modelled_call_share": round(sum(int(r.get("count", 0)) for r in known) / calls_total, 3) if calls_total else 0.0, - } - record["modelled_seen"] = [{k: r[k] for k in ("name", "count") if k in r} for r in known[: args.top]] - wanted = ("name", "spelling", "count", "nearSource", "file", "line") - - # A test helper and a build script are not product API. The layout facts already know which paths those - # are, and on the first measured subject a Gruntfile and a Cypress suite held four of the five most - # frequent candidates -- ranked first, they would have buried the two real leads beneath them. - def product(row: dict[str, Any]) -> bool: - where = str(row.get("file") or "") - return bool(where) and not is_test_path(where) and Path(where).name not in _BUILD_SCRIPTS - - ranked = [{k: r[k] for k in wanted if k in r} for r in unknown] - record["candidates"] = [r for r in ranked if product(r)][: args.top] - record["candidates_in_test_or_build"] = [r for r in ranked if not product(r)][:20] - record["candidates_near_a_source"] = [r for r in record["candidates"] if int(r.get("nearSource", 0)) > 0][:20] - save() - cleanup = getattr(graph, "cleanup", None) - if callable(cleanup): - cleanup() - backend.close_session() - print(json.dumps({"subject": args.name, "language": args.language, "coverage": record["coverage"]}), flush=True) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docs/architecture.md b/docs/architecture.md index a4b3c49..7dcd1e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,7 +16,7 @@ The CLI is thin. `HarnessRuntime` owns the lifecycle. It writes an event trace f - `quick` runs STATIC; - `standard` adds MAP; -- `deep` adds REGRESS. +- `deep` requests the REGRESS stage, which was retired on 2026-10-13: the scan records a `regress_retired` degradation and otherwise equals `standard`. Execution proof now comes from the search-with-proof arm. ``` policy_load ─▶ ruleset_load ─▶ policy_check (fail loud on an ungoverned CWE) @@ -31,7 +31,7 @@ policy_load ─▶ ruleset_load ─▶ policy_check (fail loud on an │ obligations absence bugs: an obligated operation with no governing guard │ model one Joern CPG per repository; taint, dominance and configuration arbiters │ tool hunter only with `[models] hunter` - ─▶ REGRESS (deep) sandboxed regression snippets ─▶ verdicts.json, worth-fixing + ─▶ REGRESS (deep) retired 2026-10-13, recorded as `regress_retired` ─▶ report.md, report.sarif, manifest.json ``` @@ -67,17 +67,7 @@ nothing, the LLM sees the enumerator's candidates (at most `MAX_JUDGED_CANDIDATE Anything it affirms is reported at `suspicion`. Without a provider key, neither question is asked. Only the graph's entailed findings are reported then. -**REGRESS** (`regress/`, `sandbox/`) runs only when `docker info` succeeds. Each promoted -candidate gets a small language recipe (Python, JavaScript, C). The recipe loads the -candidate's source inside a container with: - -- no network; -- a read-only `/workspace`; -- a non-root user; -- the `[sandbox]` limits. - -Before anything runs, a structural deny-list rejects snippets that mention sockets, curl, host -networking or writes under `/workspace`. +**REGRESS** was retired on 2026-10-13. Deep mode records a `regress_retired` degradation and otherwise equals standard. Execution proof is the search-with-proof arm's job (`search/verify.py`, bubblewrap sandbox, differential runs against owned canaries). ## Evidence ladder: how a false positive is eliminated diff --git a/docs/detection-techniques.md b/docs/detection-techniques.md index 4c42c58..553b6d3 100644 --- a/docs/detection-techniques.md +++ b/docs/detection-techniques.md @@ -542,10 +542,6 @@ reports the worker JVM's heap (`census.sc:17-31`). It is the instrument check of settles whether a frontend warning mattered: if the graph holds every expected file, the warning is ignored (`backend.py:827-829`). -`vocabulary.sc` is a related query that needs no dataflow. It lists every call not covered by -the fact tables as a candidate list, "never an ontology" (`vocabulary.sc:72-74`). It feeds -`benchmarks/push/ontology_gap.py`, not the scan. - ## 5. Regions, ranking and budgets A repository is too large to ask the graph about everything. So the scan asks about regions in a @@ -706,69 +702,14 @@ explains how those signals become a decision. A tool-equipped hunter was measure overlay on a Python holdout: 12/14 against 5/14 once scoring was class-aware. That gap is in `benchmarks/measurements/2026-09-06-vibe-py-holdout-hunter-*.json`. -## 7. REGRESS: sandboxed reproduction +## 7. REGRESS: retired -```mermaid -flowchart LR - hot["Promoted hotspots, top max_candidates = 5"] --> snip["Snippet: proposed by the finding, else a load-and-compile stub"] - snip --> deny{"Structural deny-list"} - deny -- "socket, curl, docker.sock, host network, writes under /workspace" --> rej["inconclusive: safety_rejected"] - deny -- "clean" --> run["docker run: no network, read-only, tmpfs scratch, cap-drop ALL, uid 65534"] - run --> exit["Exit code and output"] - exit --> v["triggerable, not_triggerable, already_covered, inconclusive"] - v --> worth["worth_fixing if triggerable and reachable"] -``` - -REGRESS runs only in `deep` mode (`stages.py:17`) and only when `docker info` succeeds. -Otherwise the stage records `sandbox_unavailable` (`sandbox/probe.py:24-36`, `cli.py:691-693`). - -It selects the top `max_candidates = 5` hotspots by score, plus forced severity-5 reachable -inventory (`regress/candidate.py:63-88`, `config.py:157-160`). A snippet is the one a finding -proposed. If there is none, a default only loads the file (`candidate.py:153-174`): - -- Python compiles it; -- JavaScript reads it; -- C is `int main(void) - { return 0; }` (`candidate.py:153-174`). - -Recipes exist for Python, JavaScript and C on `python:3.12-alpine`, `node:22-alpine` and -`gcc:13` (`regress/recipes.py:16-52`, `candidate.py:30-34`). Any other language is -`inconclusive` with `missing_recipe`. - -The deny-list is a regex check before any container starts (`regress/safety.py:63-75`). It -rejects (`safety.py:21-60`): - -- the Docker socket; -- the word `socket`; -- the word `curl`; -- a host-network request; -- writes under `/workspace` in Python, Node, C and shell forms. - -The real boundary is the container (`sandbox/runner.py:113-137`): `--network none --read-only --tmpfs /scratch:rw,exec --memory -m --pids-limit --cap-drop ALL --security-opt no-new-privileges --user 65534:65534`. The -repository is bind-mounted read-only at `/workspace`. The runner also refuses an argv containing -`--privileged`, the Docker socket or host networking (`runner.py:182-192`). Defaults are 2048 MB, -300 s and 512 pids (`config.py:104-109`). - -Verdicts (`regress/verdict.py:10-26, 64-110`): - -- a safety rejection, missing recipe or sandbox, or timeout is `inconclusive`; -- exit 0 under a covering test is `already_covered`; -- with an oracle: `triggerable` on `oracle_fired`, else `not_triggerable`; -- without an oracle: a nonzero exit is `triggerable` with reason `sanitizer_abort`, `assertion`, - `crash` or `nonzero_exit`, and exit 0 is `not_triggerable`. - -`is_worth_fixing` requires `triggerable` and reachability `reachable` or -`inferred-file-surface` (`verdict.py:28, 59-61`). Verdicts feed the complexity ledger and -`rank_obligations`. `rank_obligations` caps every obligation finding strictly below the lowest -proven finding (`cli.py:724-727`, `rank.py:243-265`). - -**What `execution_confirmed` means today.** The rung is defined as "a sandbox reproduced it -(deferred tier, memory/C first)" (`ladder.py:10, 33`). It is ordered above `model_entailed`. No -code path assigns it. REGRESS produces the four verdicts above, not a rung. Nothing maps -`triggerable` onto `Rung.EXECUTION_CONFIRMED`. The default snippets prove that a file loads, not -that a vulnerability fires. A reproduction needs a finding-proposed snippet or an oracle. Read -the rung as reserved. +The sandboxed snippet stage was retired on 2026-10-13 (degradation `regress_retired`). It proved that a file +loads, not that a vulnerability fires, and never produced a rung. Execution proof is now the search-with-proof +arm: a finding-proposed demo run under bubblewrap, three runs on the affected revision against three on the +safe one, owned canary oracles, `demonstrated` only on 3 of 3 against 0 of 3 (`search/verify.py`). +`Rung.EXECUTION_CONFIRMED` is still assigned by no code path; wiring the verify outcome onto it is the next +step in that spec. ## 8. What each technique misses @@ -781,7 +722,6 @@ the rung as reserved. | Configuration | A value computed at runtime (reported `CORROBORATED`, never safe) | `config_value.py:75-86` | | Ranking | A pinned CVE deep in a tier: PMPro's at position 390 of 4,463 under access rank, with evidence ordering still off by default | `benchmarks/ranking/baselines/2026-09-10-access-rank.json`; `scan.py:98` | | The model | Anything the enumerator did not propose (a hard ceiling on recall); the residual question is bounded to 8 candidates per region; a public fix the model may have memorised is indistinguishable from reasoning, which is why labels come from ground truth and the engine block is withheld where it tracks the fix | `candidates.py:7-10`; `pipeline.py:44`; `learn/schema.py:97-109` | -| REGRESS | Anything without a Python, JavaScript or C recipe; anything the default load-only snippet cannot trigger; no finding reaches `execution_confirmed` today | `recipes.py:16-52`; `candidate.py:153-174`; `ladder.py:33` | The three representations are kept separate on purpose. A finding carries the rung and the degradations of the representation that produced it. So a reader can tell a flow the graph diff --git a/docs/scanning.md b/docs/scanning.md index 2874193..d532681 100644 --- a/docs/scanning.md +++ b/docs/scanning.md @@ -15,7 +15,7 @@ The stages each mode requests are listed in `MODE_STAGES` in `src/openultrasast/ - `quick` runs STATIC. - `standard` adds MAP. -- `deep` adds REGRESS. +- `deep` requests the REGRESS stage, which was retired on 2026-10-13: the scan records a `regress_retired` degradation and otherwise equals `standard`. Execution proof now comes from the search-with-proof arm. ```mermaid flowchart TD @@ -29,17 +29,14 @@ flowchart TD verify --> mapq{"MAP requested?"} mapq -- "quick: no" --> out["report.md, report.sarif, manifest.json"] mapq -- "standard, deep" --> map["MAP: complexity map, semantic overlay, obligations, Joern model layer"] - map --> regq{"REGRESS requested?"} - regq -- "standard: no" --> out - regq -- "deep" --> reg["REGRESS: snippets in a Docker sandbox, verdicts.json"] - reg --> out + map --> out ``` | Mode | Adds | Needs | Without it | | --- | --- | --- | --- | | `quick` | pattern rules, entry-point hints, ranking, verification, score | nothing | - | | `standard` | MAP: complexity map, semantic overlay, authorization obligations, the Joern model layer; `fusion` (deterministic) | Joern; a provider key for the residual LLM question | `cpg_unavailable` degradation; the LLM's questions are skipped and only the graph's entailed findings are reported | -| `deep` | REGRESS: promoted candidates loaded in a sandbox with no network, read-only source, non-root | a working `docker` | `sandbox_unavailable` degradation | +| `deep` | nothing today: REGRESS was retired on 2026-10-13 and is recorded as `regress_retired`; execution proof belongs to the search-with-proof arm | - | - | Sources: `README.md` (scan modes), `src/openultrasast/cli.py` (`run_stage` calls), [Architecture](architecture.md#the-scan-pipeline). diff --git a/docs/threat-model.md b/docs/threat-model.md index d978247..048d7a5 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -11,9 +11,9 @@ It also lists the controls that keep a scan safe to run in CI and safe to share. - Standard mode also parses it into a code property graph (a graph of the program's syntax, control flow and data flow) with Joern. It uses `joern-parse`, a parser, not a build. Standard mode may send code excerpts to the configured model. - - Only `deep` mode executes target-derived code. Its REGRESS stage loads promoted candidates - inside the Docker sandbox described below. It runs only when `docker - info` succeeds (otherwise the stage is skipped and recorded as `sandbox_unavailable`). + - No scan mode executes target-derived code any more: the deep-mode REGRESS stage was retired on 2026-10-13 + and is recorded as `regress_retired`. Execution happens only in the search-with-proof arm's sandbox, + described in `.kiro/specs/search-with-proof/`. - **The ruleset and CWE policy are trusted, governed data.** They change through reviewed commits. - **Provider credentials are the operator's.** The core install (PyYAML only) makes no network @@ -78,7 +78,7 @@ malformed JSON. ### Visible degradation & determinism A scan is never silently downgraded. When an optional engine is missing, the scan falls back to its deterministic equivalent. It records a `degradations` entry in the manifest (Joern: -`cpg_unavailable`; Docker: `sandbox_unavailable`). +`cpg_unavailable`; retired snippet stage: `regress_retired`). Without a provider key, the model layer asks no question. It reports only what the graph entailed. @@ -92,18 +92,6 @@ internal hunter tools. No tool accepts a free-form command argument. ## Sandbox (deep mode) -The REGRESS stage (`regress/`, `sandbox/runner.py`) runs each snippet with `docker run`. It never -mounts the host Docker socket. - -| Control | Setting | -|---|---| -| network | `--network none`, always; a host-network argument is refused | -| source | the target bind-mounted read-only at `/workspace`; root filesystem `--read-only` | -| scratch | a tmpfs at `/scratch` for the snippet | -| user | non-root `65534:65534`, `--cap-drop ALL`, `no-new-privileges` | -| memory | `[sandbox] memory_mb`, default 2048 MB | -| pids | `[sandbox] pids_limit`, default 512 | -| timeout | `[sandbox] timeout_seconds`, default 300 s | - -A structural deny-list (`regress/safety.py`) checks each snippet before anything runs. It rejects -a snippet that mentions sockets, curl, host networking or writes under `/workspace`. +Retired on 2026-10-13 with the REGRESS stage. The search-with-proof arm runs demos under bubblewrap with +seccomp, no network unless the oracle needs a namespace, and differential runs against owned canaries; +see `src/openultrasast/search/_sandbox.py` and `.kiro/specs/search-with-proof/`. diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 0d2c8e2..6d490c5 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -34,7 +34,6 @@ from .calibration import ( write_ranking_calibrations, ) from .complexity import map as complexity_map -from .complexity.ledger import persist_verdicts from .config import ModelLayerConfig, ObligationsConfig, ResolvedConfig, RetiredConfigError, load_config, load_dotenv from .findings import StaticFinding, quick_scan_findings, uncovered_languages, write_findings from .fusion import FusionDecision, fuse_findings @@ -60,7 +59,6 @@ from .preprocess import FileTarget, preprocess_repository, write_preprocess_arti from .profiling import start as start_profiling from .provenance import fingerprint from .rank import rank_obligations, rank_targets, write_rankings -from .regress import TRIGGERABLE, CandidateVerdict, run_regression, write_verdicts from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report from .repos import ( DEFAULT_REPO_DIR, @@ -74,12 +72,10 @@ from .repos import select as select_repo from .repos import verify as verify_repo from .ruleset import DEFAULT_RULESET_DIR, load_ruleset from .run import ScanRun, create_scan_run -from .sandbox import resolve_sandbox_probe, resolve_sandbox_runner from .scoring import build_score_artifact from .semantic import ( OverlayRecord, adjudicate, - filter_promoted_hotspots, finding_from_coverage, load_facts, write_overlay, @@ -224,7 +220,7 @@ def _main(argv: list[str] | None) -> int: scan.add_argument("path", type=Path) scan.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick") scan.add_argument("--config", type=Path, default=Path("openultrasast.toml")) - scan.add_argument("--fail-on", choices=("never", "findings", "verified", "worth-fixing"), default="never") + scan.add_argument("--fail-on", choices=("never", "findings", "verified"), default="never") benchmark = subparsers.add_parser("benchmark", help="run a benchmark manifest and write scoreboard artifacts") benchmark.add_argument("manifest", type=Path) @@ -756,52 +752,9 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut "hotspot_count": len(built_map.hotspots), "heuristic_only": built_map.heuristic_only, } - verdicts_path = run.root / "verdicts.json" - wrote_verdicts = False - verdict_records: tuple[CandidateVerdict, ...] = () if Stage.REGRESS in plan.requested: - probe = resolve_sandbox_probe() - if not probe.available(): - plan = record_skip(plan, Stage.REGRESS, "sandbox_unavailable") - runtime.state["degradations"].append(skip_as_degradation(Stage.REGRESS, "sandbox_unavailable")) - else: - sandbox = resolve_sandbox_runner() - hotspots = built_map.hotspots if built_map is not None else () - hotspots = filter_promoted_hotspots(hotspots, overlay_records, findings) - promoted = [ - finding - for finding in findings - if finding.finding_id in {record.proposal_id for record in overlay_records if record.disposition == "promote"} - ] - cwe_by_rule_id = {rule.rule_id: rule.cwe for rule in ruleset} - rule_cwe_by_finding = {finding.finding_id: cwe_by_rule_id.get(finding.finding_id.split(":", 1)[0], "") for finding in promoted} - languages_by_path = {target.path: target.language for target in targets} - records = runtime.run_stage( - "regress", - lambda: run_regression( - hotspots, - promoted, - max_candidates=config.regress.max_candidates, - policy=policy, - rule_cwe=rule_cwe_by_finding, - languages_by_path=languages_by_path, - repo_root=run.target, - sandbox=sandbox, - sandbox_limits=config.sandbox, - images=dict(config.regress.images), - ), - ) - write_verdicts(records, verdicts_path) - persist_verdicts(run.target / CALIBRATION_DIR / "complexity_ledger.json", records) - # Req 6.4: proof rungs live on verdicts, not on StaticFinding, so obligations re-rank once the sandbox has spoken. - proven_ids = {fid for record in records if record.verdict == TRIGGERABLE for fid in record.inventory_finding_ids} - if proven_ids and any(finding.finding_id.startswith("obligation:") for finding in findings): - findings = rank_obligations(findings, proven_ids=proven_ids) - write_findings(findings, findings_path) - verdict_records = records - wrote_verdicts = True - plan = record_completed(plan, Stage.REGRESS) - worth_fixing_payload = _worth_fixing_payload(verdict_records) if wrote_verdicts else None + runtime.state["degradations"].append(skip_as_degradation(Stage.REGRESS, "regress_retired")) + plan = record_skip(plan, Stage.REGRESS, "regress_retired") # Req 5.6: what each family could see, for the report. Absent when the model layer did not run -- # and a scan with no model layer states no limits because it made no claims. coverage_rows = model_payload.get("coverage") if isinstance(model_payload, dict) else None @@ -814,7 +767,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut verifications, redact=config.hardening.redact_secrets, complexity_map=built_map, - verdicts=verdict_records if wrote_verdicts else None, overlay=overlay_records if wrote_overlay else None, obligations=obligations_cited or None, obligations_summary=obligations_payload or None, @@ -851,7 +803,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut fusion=fusion_path if fusion_decisions else None, complexity_map=complexity_map_path if complexity_payload is not None else None, overlay=overlay_path if wrote_overlay else None, - verdicts=verdicts_path if wrote_verdicts else None, ), path=manifest_path, score=score_artifact.to_dict(), @@ -859,7 +810,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut fusion=[_fusion_summary(decision) for decision in fusion_decisions] or None, stages=stages_payload(plan), complexity=complexity_payload, - worth_fixing=worth_fixing_payload, provenance=provenance.to_dict(), obligations=obligations_payload, model=model_payload, @@ -873,7 +823,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut ranked_target_count=len(rankings), finding_count=len(findings), calibrations_applied=len(applied_calibrations), - exit_code=scan_exit_code(findings, verifications, fail_on, worth_fixing_verdicts=verdict_records), + exit_code=scan_exit_code(findings, verifications, fail_on), notices=tuple( f"language not covered: {item['language']} ({item['count']} files)" for item in runtime.state["degradations"] @@ -966,23 +916,6 @@ def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]: return hints -def _worth_fixing_payload(verdicts: tuple[CandidateVerdict, ...]) -> dict[str, object]: - worth = [record for record in verdicts if record.worth_fixing] - return { - "count": len(worth), - "verdicts": [ - { - "path": record.path, - "function_name": record.function_name, - "verdict": record.verdict, - "reason": record.reason, - "worth_fixing": True, - } - for record in worth - ], - } - - def _artifact_paths(**paths: Path | None) -> dict[str, Path]: return {name: path for name, path in paths.items() if path is not None} diff --git a/src/openultrasast/complexity/__init__.py b/src/openultrasast/complexity/__init__.py index 3a92c16..de18c49 100644 --- a/src/openultrasast/complexity/__init__.py +++ b/src/openultrasast/complexity/__init__.py @@ -6,10 +6,7 @@ from .ledger import ( apply_overlay, hotspot_key, load_ledger, - must_keep_as_candidate, - persist_verdicts, record_verdict, - select_forced_candidates, write_ledger, ) from .map import ComplexityMap, Hotspot, build_complexity_map, write_complexity_map @@ -27,10 +24,7 @@ __all__ = [ "collect_signals", "hotspot_key", "load_ledger", - "must_keep_as_candidate", - "persist_verdicts", "record_verdict", - "select_forced_candidates", "write_complexity_map", "write_ledger", ] diff --git a/src/openultrasast/complexity/ledger.py b/src/openultrasast/complexity/ledger.py index 59911ce..1d3aeb3 100644 --- a/src/openultrasast/complexity/ledger.py +++ b/src/openultrasast/complexity/ledger.py @@ -8,8 +8,6 @@ from dataclasses import asdict, dataclass, replace from pathlib import Path from typing import TYPE_CHECKING -from ..findings import StaticFinding - if TYPE_CHECKING: from .map import Hotspot @@ -19,8 +17,6 @@ VERDICT_DELTAS = { "triggerable": TRIGGERABLE_DELTA, "not_triggerable": NOT_TRIGGERABLE_DELTA, } -FORCED_CANDIDATE_SEVERITY = 5 -FORCED_CANDIDATE_REACHABILITY = frozenset({"reachable", "inferred-file-surface"}) @dataclass(frozen=True) @@ -57,27 +53,6 @@ def write_ledger(path: Path, ledger: Mapping[str, LedgerEntry | Mapping[str, obj path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") -def persist_verdicts(path: Path, verdicts: Sequence[object]) -> dict[str, LedgerEntry]: - """Record triggerable / not_triggerable outcomes into the loop-owned complexity ledger.""" - ledger = load_ledger(path) - changed = False - for item in verdicts: - verdict = str(getattr(item, "verdict", "")) - if verdict not in VERDICT_DELTAS: - continue - raw_function = getattr(item, "function_name", None) - ledger = record_verdict( - ledger, - path=str(getattr(item, "path", "")), - function_name=raw_function if isinstance(raw_function, str) else None, - verdict=verdict, - ) - changed = True - if changed: - write_ledger(path, ledger) - return ledger - - def record_verdict( ledger: Mapping[str, LedgerEntry | Mapping[str, object]], *, @@ -130,49 +105,6 @@ def apply_overlay( return tuple(sorted(adjusted, key=lambda item: (-item.score, item.path, item.function_name or ""))) -def must_keep_as_candidate( - finding: StaticFinding, - policy_severity: int | object, - reachability: str | None = None, -) -> bool: - status = finding.reachability_status if reachability is None else reachability - return _as_severity(policy_severity) >= FORCED_CANDIDATE_SEVERITY and status in FORCED_CANDIDATE_REACHABILITY - - -def select_forced_candidates( - findings: Sequence[StaticFinding], - policy: Mapping[str, object], - *, - rule_cwe: Mapping[str, str] | None = None, -) -> tuple[StaticFinding, ...]: - cwe_by_id = rule_cwe or {} - return tuple( - finding - for finding in findings - if must_keep_as_candidate(finding, _policy_severity_for(finding, policy, cwe_by_id), finding.reachability_status) - ) - - -def _policy_severity_for(finding: StaticFinding, policy: Mapping[str, object], cwe_by_id: Mapping[str, str]) -> int: - if finding.finding_id in policy: - return _as_severity(policy[finding.finding_id]) - cwe = cwe_by_id.get(finding.finding_id) - if cwe is not None and cwe in policy: - return _as_severity(policy[cwe]) - return 0 - - -def _as_severity(value: object) -> int: - if isinstance(value, bool): - return 0 - if isinstance(value, int): - return value - severity = getattr(value, "severity", None) - if isinstance(severity, bool) or not isinstance(severity, int): - return 0 - return severity - - def _coerce_entry(value: object) -> LedgerEntry | None: if isinstance(value, LedgerEntry): return value diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index 6e0d038..1f3002d 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -25,8 +25,8 @@ _UNREAD_DYNAMIC = "no dynamic or deep tier exists; deep mode will define its own _UNREAD_EVIDENCE = "evidence tiers come from the program model's ladder (model/ladder.py), not from configuration" _UNREAD_VARIANTS = "the structural variant search it configured was deleted with semantic/variants.py" _UNREAD_SANDBOX = ( - "the sandbox always runs with `--network none` and a read-only workspace (sandbox/runner.py); that is an " - "invariant the runner enforces, not a knob" + "the search sandbox enforces isolation and read-only source (search/_sandbox.py); that is an " + "invariant, not a knob" ) _UNREAD_TOP_K = "no stage read it; `[complexity] max_hunter_hotspots` is the one complexity key the scan reads" _UNREAD_EMBEDDINGS = ( @@ -156,13 +156,6 @@ class ComplexityConfig: max_hunter_hotspots: int = 8 -@dataclass(frozen=True) -class RegressConfig: - # Stage-3 candidate cap and optional per-language sandbox image pins. Isolation limits stay on SandboxConfig. - max_candidates: int = 5 - images: tuple[tuple[str, str], ...] = () - - @dataclass(frozen=True) class ModelLayerConfig: # contributor-scan: the model layer in MAP. Named for the LAYER, not the model: `ModelConfig` is already @@ -215,7 +208,6 @@ class ResolvedConfig: fusion: FusionConfig = FusionConfig() hardening: HardeningConfig = HardeningConfig() complexity: ComplexityConfig = ComplexityConfig() - regress: RegressConfig = RegressConfig() obligations: ObligationsConfig = ObligationsConfig() model: ModelLayerConfig = ModelLayerConfig() push: PushConfig = PushConfig() @@ -240,7 +232,6 @@ def load_config(config_path: Path | None = None, *, dotenv: bool = True) -> Reso fusion=_load_fusion(data.get("fusion", {})), hardening=_load_hardening(data.get("hardening", {})), complexity=_load_complexity(data.get("complexity", {})), - regress=_load_regress(data.get("regress", {})), obligations=_load_obligations(data.get("obligations", {})), model=_load_model(data.get("model", {})), push=PushConfig.from_payload(data.get("push", {})), @@ -344,24 +335,6 @@ def _load_complexity(value: object) -> ComplexityConfig: ) -def _load_regress(value: object) -> RegressConfig: - data = _section(value) - return RegressConfig( - max_candidates=_int_value(data.get("max_candidates"), 5), - images=_string_map(data.get("images")), - ) - - -def _string_map(value: object) -> tuple[tuple[str, str], ...]: - data = _section(value) - images: list[tuple[str, str]] = [] - for key, item in data.items(): - pinned = _string(item) - if pinned is not None: - images.append((str(key), pinned)) - return tuple(sorted(images)) - - def _float_value(value: object, default: float) -> float: if isinstance(value, bool): return default diff --git a/src/openultrasast/cpg/queries/vocabulary.sc b/src/openultrasast/cpg/queries/vocabulary.sc deleted file mode 100644 index f874005..0000000 --- a/src/openultrasast/cpg/queries/vocabulary.sc +++ /dev/null @@ -1,101 +0,0 @@ -// What this repository calls that the ontology does not model (contributor-scan 7.1). -// -// Five of seven families established nothing in the 2026-09 family census, and every one of them failed for -// an ontology reason with the flow engine ready and nothing to aim at: no HTTP client for a server-side -// request forgery, no collection read for an object-reference defect, no WordPress spelling for a -// deserialization. Each was found by investigating one family at a time. This answers the question directly. -// -// It needs no dataflow. The structural pass already walks every call, which is why a census that took three -// investigations can be one query. -// -// What it reports is a CANDIDATE LIST, never an ontology. A name here becomes a fact only when someone -// resolves it to a library API, states the weakness class and records provenance. Adding a name because a -// subject called it is how a general-purpose tool becomes a fixture of its own benchmark. -// -// Parameters: cpgFile, modelled (comma-separated tokens any fact table names), sources (comma-separated -// source patterns, used only to say whether a call sits in a method that reads untrusted input). -// Output: fenced JSON array of {name, spelling, count, modelled, nearSource, file, line}. -@main def exec(cpgFile: String, modelled: String = "", sources: String = "") = { - importCpg(cpgFile) - - def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList - - val modelledTokens = split(modelled) - val sourcePatterns = split(sources) - - def boundedPattern(t: String) = - java.util.regex.Pattern.compile("(? (t, boundedPattern(t))) - - // The callee, which is the only part of a call that names what is being called. Matching a whole node's - // text is the defect this project has now fixed six times. - def calleeText(code: String): String = { - val open = code.indexOf('(') - val head = if (open < 0) code else code.substring(0, open) - head.trim.takeRight(120) - } - - // Names the FRONTEND synthesised are not names the code calls. jssrc2cpg lowers a destructuring to - // `_tmp_3.push` and an array literal to `__ecma.Array.factory`, and on the first measured subject those - // were the two most frequent candidates in the list -- noise at the top of a worklist is how a worklist - // stops being read. - def synthesised(name: String): Boolean = - name.startsWith("<") || name.startsWith("__ecma") || name.matches("^_tmp_?[0-9]*\\..*") || name.startsWith("_tmp") - - val calls = cpg.call.filterNot(c => c.name.startsWith(" - sourceMemo.getOrElseUpdate( - m.fullName, - m.ast.isCall.exists(x => sourcePatterns.exists(p => boundedPattern(p).matcher(x.code).find())) - ) - } - - def isModelled(c: io.shiftleft.codepropertygraph.generated.nodes.Call): Boolean = { - val callee = calleeText(c.code) - patterns.exists { case (token, pattern) => c.name == token || pattern.matcher(callee).find() } - } - - case class Row(name: String, spelling: String, modelledHere: Boolean, near: Boolean, file: String, line: Int) - - val rows = calls.map { c => - Row( - c.name, - calleeText(c.code), - isModelled(c), - nearSource(c), - c.file.name.l.headOption.getOrElse(""), - c.lineNumber.getOrElse(-1) - ) - } - - val grouped = rows.groupBy(_.name).toList.map { case (name, items) => - val first = items.head - ujson.Obj( - "name" -> name, - // The commonest qualified spelling, because `get` and `needle.get` are different leads. - "spelling" -> items.groupBy(_.spelling).toList.maxBy(_._2.size)._1, - "count" -> items.size, - "modelled" -> items.exists(_.modelledHere), - "nearSource" -> items.count(_.near), - "file" -> first.file, - "spellings" -> items.map(_.spelling).distinct.size, - "line" -> first.line - ) - } - - // Unmodelled first and by frequency, because that ordering IS the worklist. - val ordered = grouped.sortBy(o => (o("modelled").bool, -o("count").num.toInt)) - - println("---OUSAST-CPG-BEGIN---") - println(ujson.write(ujson.Arr(ordered: _*))) - println("---OUSAST-CPG-END---") -} diff --git a/src/openultrasast/hunter.py b/src/openultrasast/hunter.py index fd3bf8c..5009f64 100644 --- a/src/openultrasast/hunter.py +++ b/src/openultrasast/hunter.py @@ -21,7 +21,6 @@ class HunterBudget: max_targets: int max_findings_per_target: int retrieval_budget_chars: int - skill_budget_chars: int @dataclass(frozen=True) @@ -30,7 +29,6 @@ class HunterTask: ranking: RankingScore tier: HunterTier budget: HunterBudget - selected_skills: list[str] retrieval_context: str @@ -40,7 +38,6 @@ class HunterTrajectory: target_path: str tier: HunterTier priority: float - selected_skills: list[str] retrieval_context_chars: int findings: list[str] status: str @@ -53,9 +50,9 @@ class HunterPoolResult: DEFAULT_BUDGETS = { - "A": HunterBudget(tier="A", max_targets=20, max_findings_per_target=20, retrieval_budget_chars=6000, skill_budget_chars=2500), - "B": HunterBudget(tier="B", max_targets=40, max_findings_per_target=10, retrieval_budget_chars=4000, skill_budget_chars=1500), - "C": HunterBudget(tier="C", max_targets=80, max_findings_per_target=5, retrieval_budget_chars=2000, skill_budget_chars=800), + "A": HunterBudget(tier="A", max_targets=20, max_findings_per_target=20, retrieval_budget_chars=6000), + "B": HunterBudget(tier="B", max_targets=40, max_findings_per_target=10, retrieval_budget_chars=4000), + "C": HunterBudget(tier="C", max_targets=80, max_findings_per_target=5, retrieval_budget_chars=2000), } @@ -93,7 +90,6 @@ def schedule_hunter_tasks( ranking=ranking, tier=tier, budget=budget, - selected_skills=select_skill_snippets(target), retrieval_context=retrieval_context_by_path.get(target.path, "")[: budget.retrieval_budget_chars], ) ) @@ -134,13 +130,6 @@ def build_retrieval_context_by_path(root: Path, targets: list[FileTarget], *, ma return contexts -def select_skill_snippets(target: FileTarget, *, max_snippets: int = 4) -> list[str]: - """Route the target to its most relevant Trail of Bits skills (ids), via the skill index.""" - from .skills import route_skill_ids - - return route_skill_ids(target, max_snippets=max_snippets) - - def _run_hunter_task( root: Path, scan_id: str, @@ -154,7 +143,6 @@ def _run_hunter_task( target_path=task.target.path, tier=task.tier, priority=task.ranking.priority, - selected_skills=task.selected_skills, retrieval_context_chars=len(task.retrieval_context), findings=[finding.finding_id for finding in findings], status="completed", diff --git a/src/openultrasast/index.py b/src/openultrasast/index.py deleted file mode 100644 index 01eb221..0000000 --- a/src/openultrasast/index.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Text chunking for the skill router (`skills.py`). - -The vector index that once lived beside the chunker (`VectorIndex`, `build_vector_index`, `query_vector_index`, -`build_retrieval_package`) and the `ousast index` subcommand that fed it were removed on 2026-10-02: nothing ever -read the `chunks.json` they wrote. The decision engine's embeddings live in `learn/embeddings.py`. -""" - -from __future__ import annotations - -import hashlib -from dataclasses import dataclass - -VALID_NAMESPACES = {"repo_code", "docs", "static_findings", "mechanisms", "skills", "traces"} - - -@dataclass(frozen=True) -class CodeChunk: - chunk_id: str - namespace: str - path: str - language: str - start_line: int - end_line: int - text: str - metadata: dict[str, str | int | bool] - - -def chunk_text_namespace( - *, - namespace: str, - path: str, - text: str, - metadata: dict[str, str | int | bool] | None = None, - max_lines: int = 80, -) -> list[CodeChunk]: - if namespace not in VALID_NAMESPACES: - raise ValueError(f"unsupported namespace: {namespace}") - if max_lines < 1: - raise ValueError("max_lines must be at least 1") - lines = text.splitlines() - chunks: list[CodeChunk] = [] - for start in range(0, len(lines), max_lines): - chunk_lines = lines[start : start + max_lines] - if not any(line.strip() for line in chunk_lines): - continue - start_line = start + 1 - end_line = start + len(chunk_lines) - chunk_text = "\n".join(chunk_lines) - chunk_metadata: dict[str, str | int | bool] = { - "path": path, - "namespace": namespace, - "start_line": start_line, - "end_line": end_line, - } - if metadata: - chunk_metadata.update({key: value for key, value in metadata.items() if isinstance(value, str | int | bool)}) - chunks.append( - CodeChunk( - chunk_id=_chunk_id(f"{namespace}:{path}", start_line, end_line, chunk_text), - namespace=namespace, - path=path, - language=str(chunk_metadata.get("language", "text")), - start_line=start_line, - end_line=end_line, - text=chunk_text, - metadata=chunk_metadata, - ) - ) - return chunks - - -def _chunk_id(path: str, start_line: int, end_line: int, text: str) -> str: - digest = hashlib.sha256(f"{path}:{start_line}:{end_line}:{text}".encode()).hexdigest()[:16] - return f"{path}:{start_line}-{end_line}:{digest}" diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py deleted file mode 100644 index 7636a04..0000000 --- a/src/openultrasast/model/audit.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Module audit: what is load-bearing, what stands alone, and what is orphaned (Req 3). - -The tool outgrew itself — 99 modules and 832 tests at the point this feature began, a large share of them -serving an architecture its own measurements retired. This classifier exists so outgrowth is visible and -checkable rather than a thing someone notices two specs later. - -A module is: - - load_bearing reachable by imports from a gate, the CLI, or a module run with ``python -m`` - standalone_capability reachable the same way, but its own entry point and users, not on the scan path - orphaned not reachable from any of those -- it should not be in the tree - -Reachability decides the class. ``SPEC_OWNED`` and ``STANDALONE`` only supply the reason text: being named -by a spec does not keep a module nobody imports alive. - -Pure stdlib, no imports of the package it audits: it reads the import graph with ``ast`` so a broken module -is still classifiable. -""" - -from __future__ import annotations - -import ast -from collections.abc import Iterable, Mapping -from dataclasses import dataclass -from pathlib import Path - -PACKAGE = "openultrasast" - -# Entry points a module can be reachable from. The three gates are the contract; the CLI is the product. -ROOTS = ("gate", "map_gate", "pair_gate", "cli", "__main__") - -# Reason text for reachable modules, by the requirement that owns them. Not a classification: see audit(). -SPEC_OWNED = { - "push.cache": "pre-push-safety-net task 6.2: bounded artifact publication and leased eviction; runner integration in 6.4", - "push.report": "pre-push-safety-net Req 5.1/5.2/5.5/7.3: compact output and complete artifacts; runner integration pending", - "push.policy": "pre-push-safety-net Req 2.4/3.1/3.4/4.1: targeted same-semantics base comparison; replay integration pending", - "push.snapshot": "pre-push-safety-net Req 1.1/1.3/1.4/7.1: local immutable ref resolution; runner integration pending", - "contracts": "pre-push-safety-net Req 7.3: strict shared manifest serialization, independent of push", - "model.contracts": "pre-push-safety-net Req 4.1/7.3: generic deadline, change and exact scope contracts; integration pending", - "cpg.artifact": "pre-push-safety-net Req 4.3/7.3: backend graph identity and live lease contract; validation integration pending", - "push": "pre-push-safety-net Req 5.5/7.3: experimental push contract package; no hook installed", - "push.contracts": "pre-push-safety-net Req 5.5/7.3: immutable comparisons and independent result statuses; runner pending", - "model.specs": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by", - "model.endpoint": "model-grounded-detection Req 8: the bounded model client the hunter, the plane and the CLI share", - "model.audit": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan", - "model.ladder": "model-grounded-detection Req 5: the evidence ladder every finding carries", - "model.taint": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter", - "model.dominance": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash", - "model.config_value": "model-grounded-detection Req 6.3: constant abstraction for the configuration families", - "model.report": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", - "model.pipeline": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes", - "model.regions": "contributor-scan Req 2: regions from entry points, with the families each admits", - "model.scan": "contributor-scan Req 3: the repository driver -- one CPG, one budget, ranked spend", - "cpg": "model-grounded-detection Req 4: the CPG seam package", - "cpg.backend": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", - "cpg.capability": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", - "plane": "ai-service-plane Req 1-4: the service plane package; runner and reconciler integration in tasks 3-4", - "plane.budget": "ai-service-plane Req 2.3/2.4: metered chat client with usd and calls ceilings; task integration in tasks 3, 6", - "plane.manifests": "ai-service-plane Req 1.1-1.4: ax-shaped Task/Workspace/Model and the Run kind, validated before anything runs", - "plane.tasks": "ai-service-plane Req 4.1: the task entrypoint package the runner imports by name; runner integration pending (task 3)", - "plane.tasks.repo_facts": "ai-service-plane Req 5.1/5.2: source-only functions and callers per file; runner integration pending", - "plane.tasks.verify": "ai-service-plane Req 5.3/2.1-2.3/7.2: the batched hunt with known callers, metered and resumable per file hunt", - "plane.tasks.agree": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match", - "plane.reconciler": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`", - "plane.generate": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set", - "plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, S3), ingest and fact reuse; `ousast plane remember`", - "plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free", - "plane.tasks.alerts": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free", - "plane.engine_alerts": "harnessx-removal Req 6.3: engine alerts where quick mode has no rules (PHP), on the host", - "plane.tasks.loop": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps", - "plane.tasks.features": "learned-decision-engine Req 1.1-1.3/7.2: one allow-listed feature record per candidate, model-free", - "plane.tasks.roles": ( - "learned-decision-engine Req 8.1: model role classification per file chunk (model_sinks.py moved in); " - "its Run template comes with the harvest (task 5)" - ), - "learn.program": ( - "learned-decision-engine Req 3.1/3.3: the AI classifier program (signature, Retrieve -> Classify(k), response " - "cache); `ousast learn compile` (task 6.5) and the decide task (task 8) are its callers" - ), - "plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`", - "plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory", -} - -# Reachable modules that stand alone: their own CLI subcommand or external protocol, off the scan path. -STANDALONE = { - "search.demo_replay": "benchmarks standalone: no-model stored demo replay; python -m benchmarks.search.demo_replay", - "search.coordinator": ( - "search-with-proof Req 1/2/4: injected search executor and offline board checkpoint inspector; agent worker pending" - ), - "push_scoring": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring", - "push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs", - "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", - "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", - "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", -} - - -@dataclass(frozen=True) -class ModuleAudit: - module: str - classification: str # load_bearing | standalone_capability | orphaned - reason: str - importers: tuple[str, ...] - - -def module_name(path: Path, root: Path) -> str: - relative = path.relative_to(root).with_suffix("") - parts = [part for part in relative.parts if part != "__init__"] - return ".".join(parts) - - -def import_graph(root: Path) -> dict[str, set[str]]: - """``module -> the modules it imports`` for every module under ``root``, resolved to package-relative names.""" - graph: dict[str, set[str]] = {} - for path in sorted(root.rglob("*.py")): - if "__pycache__" in path.parts: - continue - name = module_name(path, root) - # For a package's `__init__`, the containing package *is* this module, so `from .x import y` resolves to - # `.x`; for a plain module it resolves against the parent. Getting this backwards made every module - # a package re-exports look unimported. - is_package = path.name == "__init__.py" - imported: set[str] = set() - try: - tree = ast.parse(path.read_text()) - except (OSError, SyntaxError): - graph[name] = imported - continue - package_parts = name.split(".") if is_package else name.split(".")[:-1] - package_parts = [part for part in package_parts if part] - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - if node.level: - base = package_parts[: len(package_parts) - node.level + 1] - target = ".".join([*base, node.module]) if node.module else ".".join(base) - else: - target = node.module or "" - if target.startswith(f"{PACKAGE}."): - target = target[len(PACKAGE) + 1 :] - elif target == PACKAGE: - target = "" - else: - continue - if target: - imported.add(target) - # `from .plane import engine_alerts` imports the submodule `plane.engine_alerts`, not only the - # package; names that are not modules are dropped later because they are not in the graph. - prefix = f"{target}." if target else "" - imported.update(f"{prefix}{alias.name}" for alias in node.names if alias.name != "*") - elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name.startswith(f"{PACKAGE}."): - imported.add(alias.name[len(PACKAGE) + 1 :]) - graph[name] = imported - return graph - - -def main_modules(root: Path) -> set[str]: - """Modules with a ``__main__`` block: run by name (``python -m``, the plane runner), so they are entry points.""" - found: set[str] = set() - for path in sorted(root.rglob("*.py")): - if "__pycache__" in path.parts: - continue - try: - tree = ast.parse(path.read_text()) - except (OSError, SyntaxError): - continue - for node in tree.body: - if ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Compare) - and isinstance(node.test.left, ast.Name) - and node.test.left.id == "__name__" - and any(isinstance(c, ast.Constant) and c.value == "__main__" for c in node.test.comparators) - ): - found.add(module_name(path, root)) - return found - - -def _reachable(graph: Mapping[str, set[str]], roots: Iterable[str]) -> set[str]: - seen: set[str] = set() - stack = [root for root in roots if root in graph] - while stack: - current = stack.pop() - if current in seen: - continue - seen.add(current) - for target in graph.get(current, ()): - # an import of `a.b` also keeps the package `a` alive - for candidate in (target, target.rsplit(".", 1)[0]): - if candidate in graph and candidate not in seen: - stack.append(candidate) - return seen - - -def audit(root: Path) -> tuple[ModuleAudit, ...]: - """Classify every module under ``root``.""" - graph = import_graph(root) - importers: dict[str, set[str]] = {name: set() for name in graph} - for name, targets in graph.items(): - for target in targets: - for candidate in (target, target.rsplit(".", 1)[0]): - if candidate in importers and candidate != name: - importers[candidate].add(name) - run_by_name = main_modules(root) - live = _reachable(graph, (*ROOTS, *sorted(run_by_name))) - results: list[ModuleAudit] = [] - for name in sorted(graph): - if not name: # the package __init__ itself - continue - who = tuple(sorted(importers[name])) - listed = SPEC_OWNED.get(name) or STANDALONE.get(name) - # Reachability decides; the lists only supply reason text. A listed module nothing reaches is still - # orphaned -- the list is what hid five dead modules behind "load_bearing" until 2026-10-02. - if name not in live: - reason = "no import path from an entry point or a python -m module" - if who: - reason += f" (imported only by unreachable {', '.join(who)})" - if listed: - reason += f"; listed as: {listed}" - results.append(ModuleAudit(name, "orphaned", reason, who)) - elif name in STANDALONE: - results.append(ModuleAudit(name, "standalone_capability", STANDALONE[name], who)) - elif name in SPEC_OWNED: - results.append(ModuleAudit(name, "load_bearing", SPEC_OWNED[name], who)) - elif name in ROOTS: - results.append(ModuleAudit(name, "load_bearing", "entry point", who)) - elif name in run_by_name: - results.append(ModuleAudit(name, "load_bearing", f"python -m openultrasast.{name}", who)) - else: - results.append(ModuleAudit(name, "load_bearing", f"reachable from {', '.join(ROOTS[:4])}", who)) - return tuple(results) - - -def to_dict(results: Iterable[ModuleAudit]) -> dict[str, object]: - rows = list(results) - counts: dict[str, int] = {} - for row in rows: - counts[row.classification] = counts.get(row.classification, 0) + 1 - return { - "modules": len(rows), - "counts": dict(sorted(counts.items())), - "orphaned": [row.module for row in rows if row.classification == "orphaned"], - "standalone": {row.module: row.reason for row in rows if row.classification == "standalone_capability"}, - "rows": [ - {"module": row.module, "classification": row.classification, "reason": row.reason, "importers": list(row.importers)} - for row in rows - ], - } - - -__all__ = ["ModuleAudit", "SPEC_OWNED", "STANDALONE", "audit", "import_graph", "main_modules", "module_name", "to_dict"] - - -if __name__ == "__main__": # python -m openultrasast.model.audit [SRC] -- the manifest's regeneration command - import json - import sys - - source = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).resolve().parents[1] - print(json.dumps(to_dict(audit(source)), indent=2)) diff --git a/src/openultrasast/regress/__init__.py b/src/openultrasast/regress/__init__.py deleted file mode 100644 index 35b9705..0000000 --- a/src/openultrasast/regress/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Isolated regression snippets and verdicts.""" - -from .candidate import CandidateVerdict, RegressionRunner, run_regression, select_candidates, write_verdicts -from .recipes import recipe_for -from .safety import UnsafeSnippetError, check_snippet_safety -from .verdict import ( - ALREADY_COVERED, - INCONCLUSIVE, - MISSING_RECIPE, - NOT_TRIGGERABLE, - SAFETY_REJECTED, - TRIGGERABLE, - RegressionVerdict, - is_worth_fixing, - verdict_from_result, -) - -__all__ = [ - "ALREADY_COVERED", - "INCONCLUSIVE", - "MISSING_RECIPE", - "NOT_TRIGGERABLE", - "SAFETY_REJECTED", - "TRIGGERABLE", - "CandidateVerdict", - "RegressionRunner", - "RegressionVerdict", - "UnsafeSnippetError", - "check_snippet_safety", - "is_worth_fixing", - "recipe_for", - "run_regression", - "select_candidates", - "verdict_from_result", - "write_verdicts", -] diff --git a/src/openultrasast/regress/candidate.py b/src/openultrasast/regress/candidate.py deleted file mode 100644 index d663c38..0000000 --- a/src/openultrasast/regress/candidate.py +++ /dev/null @@ -1,251 +0,0 @@ -"""Candidate selection and safety-gated regression execution.""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Mapping, Sequence -from dataclasses import asdict, dataclass, replace -from pathlib import Path - -from ..complexity.ledger import hotspot_key, select_forced_candidates -from ..complexity.map import Hotspot -from ..config import SandboxConfig -from ..findings import StaticFinding -from ..preprocess import LANGUAGE_BY_EXTENSION -from ..sandbox import SandboxJob, SandboxResult, SandboxRunner -from ..sandbox.runner import WORKSPACE_MOUNT -from .recipes import recipe_for -from .safety import UnsafeSnippetError, check_snippet_safety -from .verdict import ( - INCONCLUSIVE, - MISSING_RECIPE, - NOT_TRIGGERABLE, - SAFETY_REJECTED, - TRIGGERABLE, - RegressionVerdict, - is_worth_fixing, - verdict_from_result, -) - -DEFAULT_IMAGES = { - "python": "python:3.12-alpine", - "javascript": "node:22-alpine", - "c": "gcc:13", -} - -__all__ = [ - "INCONCLUSIVE", - "MISSING_RECIPE", - "NOT_TRIGGERABLE", - "SAFETY_REJECTED", - "TRIGGERABLE", - "CandidateVerdict", - "RegressionRunner", - "RegressionVerdict", - "run_regression", - "select_candidates", - "snippet_for", - "write_verdicts", -] - - -@dataclass(frozen=True) -class CandidateVerdict: - path: str - function_name: str | None - language: str - verdict: str - reason: str - inventory_finding_ids: tuple[str, ...] - worth_fixing: bool = False - - -def select_candidates( - hotspots: Sequence[Hotspot], - findings: Sequence[StaticFinding], - *, - max_candidates: int, - policy_severity_by_id: Mapping[str, object] | None = None, - rule_cwe: Mapping[str, str] | None = None, -) -> tuple[Hotspot, ...]: - """Select min(max_candidates, ranked hotspots) ∪ sev-5 reachable inventory. - - The cap applies to the ranked hotspot slice. Forced policy-severity-5 - reachable (or inferred-file-surface) inventory stays in the set even when a - ledger overlay demoted its hotspot out of that slice. - """ - ranked = tuple(sorted(hotspots, key=lambda item: (-item.score, item.path, item.function_name or ""))) - selected = list(ranked[: max(max_candidates, 0)]) - seen = {hotspot_key(item.path, item.function_name) for item in selected} - extras: list[Hotspot] = [] - for finding in select_forced_candidates(findings, policy_severity_by_id or {}, rule_cwe=rule_cwe): - hotspot = _hotspot_for_finding(finding, ranked) or _hotspot_from_forced_finding(finding) - key = hotspot_key(hotspot.path, hotspot.function_name) - if key in seen: - continue - seen.add(key) - extras.append(hotspot) - return tuple(selected + extras) - - -def _hotspot_for_finding(finding: StaticFinding, hotspots: Sequence[Hotspot]) -> Hotspot | None: - for hotspot in hotspots: - if finding.finding_id in hotspot.inventory_finding_ids: - return hotspot - key = hotspot_key(finding.path, finding.function_name) - for hotspot in hotspots: - if hotspot_key(hotspot.path, hotspot.function_name) == key: - return hotspot - return None - - -def hotspot_from_finding(finding: StaticFinding, *, rationale: str) -> Hotspot: - """A zero-score hotspot wrapping one finding: forced sev-5 inventory or a variant of a known mechanism.""" - return Hotspot( - path=finding.path, - function_name=finding.function_name, - score=0.0, - band="low", - signals={}, - rationale=rationale, - test_hint=None, - inventory_finding_ids=(finding.finding_id,), - ) - - -def _hotspot_from_forced_finding(finding: StaticFinding) -> Hotspot: - return hotspot_from_finding(finding, rationale="forced reachable severity-5 inventory") - - -class RegressionRunner: - """Run a snippet in the sandbox only after the structural safety check passes.""" - - def __init__(self, sandbox: SandboxRunner) -> None: - self._sandbox = sandbox - - def run_recipe( - self, - language: str, - snippet: str, - image: str, - sandbox_limits: SandboxConfig, - *, - repo_root: Path, - oracle: Callable[[SandboxResult], bool] | None = None, - ) -> RegressionVerdict: - job = recipe_for(language, snippet, image, sandbox_limits, repo_root=repo_root) - if job is None: - return verdict_from_result(recipe_missing=True) - return self.run_snippet(snippet, job, oracle=oracle) - - def run_snippet( - self, snippet: str, job: SandboxJob | None, *, oracle: Callable[[SandboxResult], bool] | None = None - ) -> RegressionVerdict: - if job is None: - return verdict_from_result(recipe_missing=True) - try: - check_snippet_safety(snippet) - except UnsafeSnippetError: - return verdict_from_result(safety_rejected=True) - return verdict_from_result(self._sandbox.run(job), oracle=oracle) - - -def snippet_for(language: str, path: str, function_name: str | None) -> str: - """Render a scratch snippet that loads the candidate source inside the sandbox.""" - relative = path.replace("\\", "/") - workspace_path = f"{WORKSPACE_MOUNT}/{relative}" - identity = function_name or relative - if language == "python": - return ( - f"# candidate {identity}\n" - "from pathlib import Path\n" - f"source = Path({workspace_path!r}).read_text()\n" - f"compile(source, {relative!r}, 'exec')\n" - ) - if language == "javascript": - return ( - f"// candidate {identity}\n" - "const fs = require('fs');\n" - f"const source = fs.readFileSync({workspace_path!r}, 'utf8');\n" - "if (!source.length) { process.exit(1); }\n" - ) - if language == "c": - return f"/* candidate {identity} */\nint main(void) {{ return 0; }}\n" - return "" - - -def run_regression( - hotspots: Sequence[Hotspot], - findings: Sequence[StaticFinding], - *, - max_candidates: int, - policy: Mapping[str, object], - rule_cwe: Mapping[str, str], - languages_by_path: Mapping[str, str], - repo_root: Path, - sandbox: SandboxRunner, - sandbox_limits: SandboxConfig, - images: Mapping[str, str], -) -> tuple[CandidateVerdict, ...]: - """Select candidates under the cap, run recipes in the sandbox, and return verdicts.""" - selected = select_candidates( - hotspots, - findings, - max_candidates=max_candidates, - policy_severity_by_id=policy, - rule_cwe=rule_cwe, - ) - runner = RegressionRunner(sandbox) - findings_by_id = {finding.finding_id: finding for finding in findings} - records: list[CandidateVerdict] = [] - for hotspot in selected: - language = _language_for(hotspot.path, languages_by_path) - snippet = _proposed_snippet(hotspot, findings) or snippet_for(language, hotspot.path, hotspot.function_name) - image = images.get(language) or DEFAULT_IMAGES.get(language, "ousast-missing-image") - mapped = runner.run_recipe(language, snippet, image, sandbox_limits, repo_root=repo_root) - reachability = _reachability_for(hotspot, findings_by_id) - mapped = replace(mapped, worth_fixing=is_worth_fixing(mapped.verdict, reachability)) - records.append( - CandidateVerdict( - path=hotspot.path, - function_name=hotspot.function_name, - language=language, - verdict=mapped.verdict, - reason=mapped.reason, - inventory_finding_ids=hotspot.inventory_finding_ids, - worth_fixing=mapped.worth_fixing, - ) - ) - return tuple(records) - - -def write_verdicts(verdicts: Sequence[CandidateVerdict], path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - payload = {"verdicts": [asdict(item) for item in verdicts]} - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -def _proposed_snippet(hotspot: Hotspot, findings: Sequence[StaticFinding]) -> str | None: - for finding in findings: - snippet = finding.proposed_snippet - if not snippet: - continue - if finding.path == hotspot.path or finding.finding_id in hotspot.inventory_finding_ids: - return snippet - return None - - -def _language_for(path: str, languages_by_path: Mapping[str, str]) -> str: - if path in languages_by_path: - return languages_by_path[path] - return LANGUAGE_BY_EXTENSION.get(Path(path).suffix.lower(), "unknown") - - -def _reachability_for(hotspot: Hotspot, findings_by_id: Mapping[str, StaticFinding]) -> str: - statuses = [ - findings_by_id[finding_id].reachability_status for finding_id in hotspot.inventory_finding_ids if finding_id in findings_by_id - ] - for status in statuses: - if is_worth_fixing(TRIGGERABLE, status): - return status - return statuses[0] if statuses else "unknown" diff --git a/src/openultrasast/regress/recipes.py b/src/openultrasast/regress/recipes.py deleted file mode 100644 index fbcd7bc..0000000 --- a/src/openultrasast/regress/recipes.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Language command recipes for isolated regression snippets.""" - -from __future__ import annotations - -from pathlib import Path - -from ..config import SandboxConfig -from ..sandbox.runner import SCRATCH_MOUNT, WORKSPACE_MOUNT, SandboxJob - -_PYTHON_FILE = "case.py" -_JS_FILE = "case.js" -_C_FILE = "case.c" -_C_BIN = "case" - - -def recipe_for( - language: str, - snippet: str, - image: str, - sandbox_limits: SandboxConfig, - *, - repo_root: Path, -) -> SandboxJob | None: - """Return a sandbox job for `language`, or None when no recipe exists.""" - key = language.strip().lower() - if key == "python": - return _job( - image, - sandbox_limits, - repo_root, - command=("env", f"PYTHONPATH={WORKSPACE_MOUNT}", "python", f"{SCRATCH_MOUNT}/{_PYTHON_FILE}"), - scratch_files={_PYTHON_FILE: snippet}, - ) - if key == "javascript": - return _job( - image, - sandbox_limits, - repo_root, - command=("env", f"NODE_PATH={WORKSPACE_MOUNT}", "node", f"{SCRATCH_MOUNT}/{_JS_FILE}"), - scratch_files={_JS_FILE: snippet}, - ) - if key == "c": - source = f"{SCRATCH_MOUNT}/{_C_FILE}" - binary = f"{SCRATCH_MOUNT}/{_C_BIN}" - return _job( - image, - sandbox_limits, - repo_root, - command=("sh", "-c", f"cc {source} -o {binary} && {binary}"), - scratch_files={_C_FILE: snippet}, - ) - return None - - -def _job( - image: str, - sandbox_limits: SandboxConfig, - repo_root: Path, - *, - command: tuple[str, ...], - scratch_files: dict[str, str], -) -> SandboxJob: - return SandboxJob( - image=image, - command=command, - repo_root=repo_root, - scratch_files=scratch_files, - timeout_seconds=sandbox_limits.timeout_seconds, - memory_mb=sandbox_limits.memory_mb, - pids_limit=sandbox_limits.pids_limit, - ) diff --git a/src/openultrasast/regress/safety.py b/src/openultrasast/regress/safety.py deleted file mode 100644 index d8dfb26..0000000 --- a/src/openultrasast/regress/safety.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Structural deny-list for untrusted regression snippets. - -Snippets are data for `/scratch` inside the sandbox. Mentions of sockets, curl, -host networking, or writes under the source mount (`/workspace`) are rejected -before any `SandboxRunner.run` call. -""" - -from __future__ import annotations - -import re - -from ..sandbox.runner import WORKSPACE_MOUNT - -_WORKSPACE = re.escape(WORKSPACE_MOUNT) - - -class UnsafeSnippetError(ValueError): - """Raised when a snippet fails the structural sandbox safety check.""" - - -_HOST_NETWORK = re.compile( - r"--net(?:work)?(?:\s*=\s*|\s+|['\"]\s*,\s*['\"])host\b", - re.IGNORECASE, -) -_SOCKET_WORD = re.compile(r"\bsocket\b", re.IGNORECASE) -_CURL_WORD = re.compile(r"\bcurl\b", re.IGNORECASE) -_OPEN_WORKSPACE_WRITE = re.compile( - rf"open\s*\(\s*[fFrRbBuU]*['\"]{_WORKSPACE}[^'\"]*['\"]\s*,\s*(?:mode\s*=\s*)?[fFrRbBuU]*['\"][^'\"]*[wax+]", - re.IGNORECASE | re.DOTALL, -) -_PATH_WORKSPACE_WRITE = re.compile( - rf"Path\s*\(\s*['\"]{_WORKSPACE}[^'\"]*['\"]\s*\)\s*\.\s*" - r"(?:write_text|write_bytes|touch|mkdir|unlink|rmdir|replace|chmod|rename|open)\b", - re.IGNORECASE | re.DOTALL, -) -_FS_WORKSPACE_WRITE = re.compile( - r"(?:writeFile(?:Sync)?|appendFile(?:Sync)?|mkdir(?:Sync)?|rm(?:dir)?(?:Sync)?|" - rf"unlink(?:Sync)?|rename(?:Sync)?)\s*\(\s*['\"]{_WORKSPACE}", - re.IGNORECASE, -) -_FOPEN_WORKSPACE_WRITE = re.compile( - rf"fopen\s*\(\s*['\"]{_WORKSPACE}[^'\"]*['\"]\s*,\s*['\"][^'\"]*[wax+]", - re.IGNORECASE | re.DOTALL, -) -_SHUTIL_OS_WORKSPACE_WRITE = re.compile( - r"(?:shutil\.(?:copy(?:2)?|copyfile|copytree|move|rmtree)|" - r"os\.(?:remove|unlink|replace|rename|mkdir|makedirs|rmdir|chmod))" - rf"\s*\([^;]{{0,240}}{_WORKSPACE}", - re.IGNORECASE | re.DOTALL, -) -_REDIRECT_WORKSPACE = re.compile(rf"(?:>>|>)\s*['\"]?{_WORKSPACE}\b", re.IGNORECASE) - -_WORKSPACE_WRITES = ( - _OPEN_WORKSPACE_WRITE, - _PATH_WORKSPACE_WRITE, - _FS_WORKSPACE_WRITE, - _FOPEN_WORKSPACE_WRITE, - _SHUTIL_OS_WORKSPACE_WRITE, - _REDIRECT_WORKSPACE, -) - - -def check_snippet_safety(snippet: str) -> None: - """Return None when `snippet` is structurally safe; raise otherwise.""" - lowered = snippet.lower() - if "docker.sock" in lowered: - raise UnsafeSnippetError("snippet mentions the Docker socket") - if _SOCKET_WORD.search(snippet) is not None: - raise UnsafeSnippetError("snippet mentions socket") - if _CURL_WORD.search(snippet) is not None: - raise UnsafeSnippetError("snippet mentions curl") - if _HOST_NETWORK.search(snippet) is not None: - raise UnsafeSnippetError("snippet requests host network") - if _writes_under_workspace(snippet): - raise UnsafeSnippetError(f"snippet writes under {WORKSPACE_MOUNT}") - - -def _writes_under_workspace(snippet: str) -> bool: - if WORKSPACE_MOUNT not in snippet.lower(): - return False - return any(pattern.search(snippet) is not None for pattern in _WORKSPACE_WRITES) diff --git a/src/openultrasast/regress/verdict.py b/src/openultrasast/regress/verdict.py deleted file mode 100644 index 637df3a..0000000 --- a/src/openultrasast/regress/verdict.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Map sandbox outcomes to verdicts and the worth-fixing gate.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass - -from ..sandbox import SandboxResult - -TRIGGERABLE = "triggerable" -NOT_TRIGGERABLE = "not_triggerable" -ALREADY_COVERED = "already_covered" -INCONCLUSIVE = "inconclusive" - -MISSING_RECIPE = "missing_recipe" -SAFETY_REJECTED = "safety_rejected" -SANDBOX_UNAVAILABLE = "sandbox_unavailable" -TIMEOUT = "timeout" -COVERED_TEST_PASSED = "covered_test_passed" -ASSERTION = "assertion" -SANITIZER_ABORT = "sanitizer_abort" -CRASH = "crash" -NONZERO_EXIT = "nonzero_exit" -EXIT_ZERO = "exit_zero" -ORACLE_FIRED = "oracle_fired" -ORACLE_QUIET = "oracle_quiet" - -WORTH_FIXING_REACHABILITY = frozenset({"reachable", "inferred-file-surface"}) - -_SANITIZER_MARKERS = ( - "AddressSanitizer", - "UndefinedBehaviorSanitizer", - "ThreadSanitizer", - "MemorySanitizer", - "LeakSanitizer", -) -_ASSERTION_MARKERS = ( - "AssertionError", - "Assertion failed", - "assertion failed", - "assert failed", -) -_CRASH_MARKERS = ( - "Fatal Python error", - "Segmentation fault", - "SIGSEGV", - "SIGABRT", - "Aborted", -) - - -@dataclass(frozen=True) -class RegressionVerdict: - verdict: str - reason: str = "" - worth_fixing: bool = False - - -def is_worth_fixing(verdict: str, reachability: str) -> bool: - """True iff the candidate is triggerable on a reachable or inferred-file-surface finding.""" - return verdict == TRIGGERABLE and reachability in WORTH_FIXING_REACHABILITY - - -def verdict_from_result( - result: SandboxResult | None = None, - *, - recipe_missing: bool = False, - sandbox_missing: bool = False, - safety_rejected: bool = False, - covering_test: bool = False, - oracle: Callable[[SandboxResult], bool] | None = None, -) -> RegressionVerdict: - """Map a sandbox observation onto one of the four regression verdicts. - - ``oracle`` is how a family says what "the vulnerability fired" means for it: when one is given it - decides triggerability instead of the exit code, because a crash and a demonstrated injection are - not the same evidence. An inconclusive run stays inconclusive either way. - """ - if recipe_missing: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE) - if sandbox_missing: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE) - if safety_rejected: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED) - if result is None: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE) - if result.timed_out: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=TIMEOUT) - if covering_test and result.exit_code == 0: - return RegressionVerdict(verdict=ALREADY_COVERED, reason=COVERED_TEST_PASSED) - if oracle is not None: - return ( - RegressionVerdict(verdict=TRIGGERABLE, reason=ORACLE_FIRED) - if oracle(result) - else RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=ORACLE_QUIET) - ) - if result.exit_code != 0: - return RegressionVerdict(verdict=TRIGGERABLE, reason=_nonzero_reason(result)) - return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=EXIT_ZERO) - - -def _nonzero_reason(result: SandboxResult) -> str: - text = f"{result.stdout}\n{result.stderr}".lower() - if any(marker.lower() in text for marker in _SANITIZER_MARKERS): - return SANITIZER_ABORT - if any(marker.lower() in text for marker in _ASSERTION_MARKERS): - return ASSERTION - if any(marker.lower() in text for marker in _CRASH_MARKERS): - return CRASH - return NONZERO_EXIT diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index bb139ec..7b3b6cb 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -17,7 +17,6 @@ def write_markdown_report( *, redact: bool = True, complexity_map: object | None = None, - verdicts: Sequence[object] | None = None, overlay: Sequence[object] | None = None, mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligations: Mapping[str, Mapping[str, object]] | None = None, @@ -103,8 +102,6 @@ def write_markdown_report( _append_overlay(lines, overlay) if complexity_map is not None: _append_complexity_map(lines, complexity_map) - if verdicts is not None: - _append_worth_fixing(lines, verdicts) report = "\n".join(lines).rstrip() + "\n" if redact: from .redaction import redact_secrets @@ -149,25 +146,6 @@ def _append_complexity_map(lines: list[str], complexity_map: object) -> None: lines.extend(["", rationale, ""] if rationale else [""]) -def _append_worth_fixing(lines: list[str], verdicts: Sequence[object]) -> None: - worth = [item for item in verdicts if getattr(item, "worth_fixing", False)] - lines.extend(["## Worth fixing", "", f"Count: {len(worth)}", ""]) - if not worth: - lines.extend(["No worth-fixing verdicts were produced.", ""]) - return - for item in worth: - identity = _report_identity(getattr(item, "path", ""), getattr(item, "function_name", None)) - lines.extend( - [ - f"### `{identity}`", - "", - f"- Verdict: `{getattr(item, 'verdict', '')}`", - f"- Reason: `{getattr(item, 'reason', '')}`", - "", - ] - ) - - def _report_identity(path: object, function_name: object) -> str: path_text = str(path or "") if function_name: @@ -227,7 +205,6 @@ def write_manifest( fusion: list[dict[str, object]] | None = None, stages: dict[str, object] | None = None, complexity: dict[str, object] | None = None, - worth_fixing: dict[str, object] | None = None, provenance: dict[str, object] | None = None, variants: dict[str, object] | None = None, obligations: dict[str, object] | None = None, @@ -265,8 +242,6 @@ def write_manifest( payload["stages"] = stages if complexity is not None: payload["complexity"] = complexity - if worth_fixing is not None: - payload["worth_fixing"] = worth_fixing if provenance is not None: payload["provenance"] = provenance if variants is not None: @@ -282,7 +257,6 @@ def scan_exit_code( findings: list[StaticFinding], verifications: list[VerificationResult], policy: str, - worth_fixing_verdicts: Sequence[object] | None = None, ) -> int: if policy == "never": return 0 @@ -290,8 +264,6 @@ def scan_exit_code( return 1 if findings else 0 if policy == "verified": return 1 if any(result.verified for result in verifications) else 0 - if policy == "worth-fixing": - return 1 if any(getattr(verdict, "worth_fixing", False) for verdict in worth_fixing_verdicts or ()) else 0 raise ValueError(f"unknown fail policy: {policy}") @@ -463,7 +435,7 @@ _COVERAGE_DEGRADATIONS: dict[str, str] = { "policy_invalid": "The `{stage}` policy could not be loaded, so that stage decided nothing. {detail}", "max_findings_exceeded": "Only the first {requested} of {actual} findings are reported (`max_findings`).", "hunter_model_unavailable": "The `{stage}` stage needs a model and none is available, so it was skipped.", - "sandbox_unavailable": "Docker is not available, so no finding was reproduced in the sandbox (REGRESS skipped).", + "regress_retired": "the sandboxed snippet stage was retired on 2026-10-13; execution proof comes from the search-with-proof arm", "language_not_covered": ( "language not covered: {language} ({count} files). No quick rules{engine} exist for it, so **silence about " "these files is not a clean result**." diff --git a/src/openultrasast/sandbox/__init__.py b/src/openultrasast/sandbox/__init__.py index 6728be9..f268d96 100644 --- a/src/openultrasast/sandbox/__init__.py +++ b/src/openultrasast/sandbox/__init__.py @@ -1,25 +1,5 @@ -"""Isolated Docker CLI sandbox.""" +"""Shared sandbox job and result types.""" -from .probe import FakeSandboxProbe, SandboxProbe, resolve_sandbox_probe -from .runner import ( - DockerCliRunner, - FakeSandboxRunner, - SandboxJob, - SandboxResult, - SandboxRunner, - build_docker_argv, - resolve_sandbox_runner, -) +from .runner import SandboxJob, SandboxResult -__all__ = [ - "DockerCliRunner", - "FakeSandboxProbe", - "FakeSandboxRunner", - "SandboxJob", - "SandboxProbe", - "SandboxResult", - "SandboxRunner", - "build_docker_argv", - "resolve_sandbox_probe", - "resolve_sandbox_runner", -] +__all__ = ["SandboxJob", "SandboxResult"] diff --git a/src/openultrasast/sandbox/probe.py b/src/openultrasast/sandbox/probe.py deleted file mode 100644 index 6cac98c..0000000 --- a/src/openultrasast/sandbox/probe.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Docker CLI presence and usability probe.""" - -from __future__ import annotations - -import os -import subprocess -from collections.abc import Callable -from subprocess import CompletedProcess - -PROBE_TIMEOUT_SECONDS = 5 -PROBE_ENV = "OPENULTRASAST_SANDBOX_PROBE" - -_DOCKER_INFO = ("docker", "info") -_PROBE_OFF = frozenset({"0", "off", "false", "unavailable", "no"}) -_PROBE_ON = frozenset({"1", "on", "true", "available", "yes"}) - - -class SandboxProbe: - """Report whether `docker info` succeeds without raising into the scan.""" - - def __init__(self, *, runner: Callable[..., CompletedProcess[str]] | None = None) -> None: - self._runner = runner - - def available(self) -> bool: - run = self._runner if self._runner is not None else subprocess.run - try: - result = run( - list(_DOCKER_INFO), - capture_output=True, - text=True, - timeout=PROBE_TIMEOUT_SECONDS, - check=False, - ) - return result.returncode == 0 - except Exception: - return False - - -class FakeSandboxProbe: - """Test double that reports a programmed availability without touching Docker.""" - - def __init__(self, available: bool) -> None: - self._available = available - - def available(self) -> bool: - return self._available - - -def resolve_sandbox_probe() -> SandboxProbe | FakeSandboxProbe: - """Return a real probe, or a fake when OPENULTRASAST_SANDBOX_PROBE is set.""" - flag = os.environ.get(PROBE_ENV, "").strip().lower() - if flag in _PROBE_OFF: - return FakeSandboxProbe(False) - if flag in _PROBE_ON: - return FakeSandboxProbe(True) - return SandboxProbe() diff --git a/src/openultrasast/sandbox/runner.py b/src/openultrasast/sandbox/runner.py index 4917cfe..e865d25 100644 --- a/src/openultrasast/sandbox/runner.py +++ b/src/openultrasast/sandbox/runner.py @@ -1,28 +1,9 @@ -"""Docker CLI sandbox runner that never mounts the host Docker socket.""" +"""Job and result types shared by the search sandbox and verifier.""" from __future__ import annotations -import os -import subprocess -import tempfile -from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path -from subprocess import CompletedProcess -from typing import Protocol - -NONROOT_USER = "65534:65534" -WORKSPACE_MOUNT = "/workspace" -SCRATCH_MOUNT = "/scratch" -SCRATCH_SEED_MOUNT = "/scratch-in" -RUNNER_ENV = "OPENULTRASAST_SANDBOX_RUNNER" -_COPY_SEED_THEN_EXEC = 'cp -r /scratch-in/. /scratch/ && exec "$@"' -_SEED_ARGV0 = "ousast-sandbox" -_FAKE_RUNNER = frozenset({"fake"}) - - -class SandboxRunner(Protocol): - def run(self, spec: SandboxJob) -> SandboxResult: ... @dataclass(frozen=True) @@ -43,158 +24,3 @@ class SandboxResult: stderr: str timed_out: bool - -class FakeSandboxRunner: - """Record jobs and return a programmed result without a Docker daemon.""" - - def __init__(self, result: SandboxResult | None = None) -> None: - self.jobs: list[SandboxJob] = [] - self.result = result if result is not None else SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False) - - def run(self, spec: SandboxJob) -> SandboxResult: - self.jobs.append(spec) - return self.result - - -def resolve_sandbox_runner() -> SandboxRunner: - """Return the Docker CLI runner, or a fake when OPENULTRASAST_SANDBOX_RUNNER=fake.""" - flag = os.environ.get(RUNNER_ENV, "").strip().lower() - if flag in _FAKE_RUNNER: - return FakeSandboxRunner() - return DockerCliRunner() - - -class DockerCliRunner: - """Run a job with Docker CLI flags that match the threat model.""" - - def __init__(self, *, runner: Callable[..., CompletedProcess[str]] | None = None) -> None: - self._runner = runner - - def run(self, spec: SandboxJob) -> SandboxResult: - if spec.scratch_files: - with tempfile.TemporaryDirectory(prefix="ousast-scratch-") as tmp: - seed = Path(tmp) - _write_scratch_files(seed, spec.scratch_files) - return self._invoke(_argv_with_scratch_seed(spec, seed), spec) - return self._invoke(build_docker_argv(spec), spec) - - def _invoke(self, argv: list[str], spec: SandboxJob) -> SandboxResult: - _reject_unsafe_argv(argv) - run = self._runner if self._runner is not None else subprocess.run - try: - completed = run( - argv, - capture_output=True, - text=True, - timeout=spec.timeout_seconds, - check=False, - ) - except subprocess.TimeoutExpired as exc: - return SandboxResult( - exit_code=-1, - stdout=_as_text(exc.stdout), - stderr=_as_text(exc.stderr), - timed_out=True, - ) - return SandboxResult( - exit_code=completed.returncode, - stdout=completed.stdout or "", - stderr=completed.stderr or "", - timed_out=False, - ) - - -def build_docker_argv(job: SandboxJob) -> list[str]: - argv = [*_isolation_argv(job), job.image, *job.command] - _reject_unsafe_argv(argv) - return argv - - -def _isolation_argv(job: SandboxJob) -> list[str]: - repo = str(job.repo_root.resolve()) - return [ - "docker", - "run", - "--rm", - "--init", - "--network", - "none", - "--read-only", - "--tmpfs", - f"{SCRATCH_MOUNT}:rw,exec", - "--memory", - f"{job.memory_mb}m", - "--pids-limit", - str(job.pids_limit), - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges", - "--user", - NONROOT_USER, - "--mount", - f"type=bind,src={repo},dst={WORKSPACE_MOUNT},ro", - ] - - -def _argv_with_scratch_seed(job: SandboxJob, seed: Path) -> list[str]: - argv = [ - *_isolation_argv(job), - "--mount", - f"type=bind,src={seed.resolve()},dst={SCRATCH_SEED_MOUNT},ro", - job.image, - "/bin/sh", - "-c", - _COPY_SEED_THEN_EXEC, - _SEED_ARGV0, - *job.command, - ] - _reject_unsafe_argv(argv) - return argv - - -def _write_scratch_files(root: Path, files: Mapping[str, str]) -> None: - resolved_root = root.resolve() - for relative, content in files.items(): - candidate = Path(relative) - if candidate.is_absolute(): - raise ValueError(f"scratch path {relative!r} must be relative to {SCRATCH_MOUNT}") - destination = (resolved_root / candidate).resolve() - try: - common = os.path.commonpath([str(resolved_root), str(destination)]) - except ValueError as exc: - raise ValueError(f"scratch path {relative!r} escapes {SCRATCH_MOUNT}") from exc - if common != str(resolved_root): - raise ValueError(f"scratch path {relative!r} escapes {SCRATCH_MOUNT}") - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content) - _chmod_seed_tree(resolved_root) - - -def _chmod_seed_tree(root: Path) -> None: - # Container user 65534 must be able to traverse and read the seed bind. - for dirpath, _dirnames, filenames in os.walk(root): - Path(dirpath).chmod(0o755) - for name in filenames: - (Path(dirpath) / name).chmod(0o644) - - -def _reject_unsafe_argv(argv: list[str]) -> None: - joined = " ".join(argv) - if "/var/run/docker.sock" in joined: - raise ValueError("sandbox argv must not mount the host Docker socket") - if "--privileged" in argv: - raise ValueError("sandbox argv must not be privileged") - for index, item in enumerate(argv): - if item in {"--network", "--net"} and index + 1 < len(argv) and argv[index + 1] == "host": - raise ValueError("sandbox argv must not use host network") - if item in {"--network=host", "--net=host"}: - raise ValueError("sandbox argv must not use host network") - - -def _as_text(value: str | bytes | None) -> str: - if value is None: - return "" - if isinstance(value, bytes): - return value.decode(errors="replace") - return value diff --git a/src/openultrasast/skills.py b/src/openultrasast/skills.py deleted file mode 100644 index 8461e78..0000000 --- a/src/openultrasast/skills.py +++ /dev/null @@ -1,372 +0,0 @@ -"""Trail of Bits skill index + router (Phase 14 of the retired openrouter-sast-harness plan). - -Security skills are scoped expertise, not global prompt bloat. Each skill is a small -descriptor with routing metadata (languages, tags, vulnerability classes, stages) and -a short operating checklist. The router selects the most relevant skills for a target -and stage; ``select_skill_context`` packs their checklists into a char budget; and -``skill_chunks`` chunks the guidance into the ``skills`` retrieval namespace. Nothing -here imports a model or the optional extra — it is pure routing data. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from .index import CodeChunk, chunk_text_namespace -from .preprocess import FileTarget - -# Pipeline stages a skill can apply to. -STAGES = ("map", "hunt", "verify", "fix") - -# FileTarget tags → vulnerability classes the router reasons over. -TAG_TO_VULN = { - "memory_unsafe": "memory-safety", - "parser": "parsing", - "fuzzable": "parsing", - "crypto": "crypto", - "deserialization": "deserialization", - "auth_boundary": "access-control", - "network_entry": "access-control", - "privileged": "access-control", -} - - -@dataclass(frozen=True) -class SkillDescriptor: - id: str - title: str - summary: str - checklist: str - languages: tuple[str, ...] = () # empty => language-agnostic - tags: tuple[str, ...] = () - vulnerability_classes: tuple[str, ...] = () - stages: tuple[str, ...] = () # empty => any stage - - def descriptor(self) -> dict[str, object]: - return { - "id": self.id, - "title": self.title, - "summary": self.summary, - "languages": list(self.languages), - "tags": list(self.tags), - "vulnerability_classes": list(self.vulnerability_classes), - "stages": list(self.stages), - } - - -# Curated inventory aligned with the spec's Trail of Bits skill routing examples. -SKILL_INVENTORY: tuple[SkillDescriptor, ...] = ( - SkillDescriptor( - "c-review", - "C/C++ security review", - "Manual review discipline for memory-unsafe code.", - "Audit buffer indexing, object lifetime, integer overflow before length math, and unchecked copies.", - languages=("c", "cpp", "c++"), - tags=("memory_unsafe",), - vulnerability_classes=("memory-safety",), - stages=("map", "hunt"), - ), - SkillDescriptor( - "address-sanitizer", - "AddressSanitizer", - "Detect memory-safety violations at runtime.", - "Build with -fsanitize=address; reproduce the overflow/UAF; capture the ASan trace as crash evidence.", - languages=("c", "cpp", "c++"), - tags=("memory_unsafe",), - vulnerability_classes=("memory-safety",), - stages=("hunt", "verify"), - ), - SkillDescriptor( - "libfuzzer", - "libFuzzer harnessing", - "In-process coverage-guided fuzzing of C/C++ entries.", - "Write a LLVMFuzzerTestOneInput harness over the parser entry; minimize crashing inputs.", - languages=("c", "cpp", "c++"), - tags=("memory_unsafe", "fuzzable", "parser"), - vulnerability_classes=("memory-safety", "parsing"), - stages=("hunt", "verify"), - ), - SkillDescriptor( - "harness-writing", - "Fuzz harness writing", - "Build deterministic harnesses around attacker-controlled entries.", - "Identify the smallest attacker-controlled entry; build a deterministic harness; seed a corpus.", - languages=("c", "cpp", "c++"), - tags=("memory_unsafe", "fuzzable", "parser"), - vulnerability_classes=("memory-safety", "parsing"), - stages=("hunt", "verify"), - ), - SkillDescriptor( - "fuzzing-dictionary", - "Fuzzing dictionary", - "Reach deep parser states with a format dictionary.", - "Extract format tokens/magic bytes into a fuzzing dictionary to reach deep parser states.", - tags=("parser", "fuzzable"), - vulnerability_classes=("parsing",), - stages=("verify",), - ), - SkillDescriptor( - "fuzzing-obstacles", - "Fuzzing obstacles", - "Remove coverage barriers that block the fuzzer.", - "Remove checksums/magic gates that block coverage; document each bypass.", - tags=("parser", "fuzzable"), - vulnerability_classes=("parsing",), - stages=("verify",), - ), - SkillDescriptor( - "atheris", - "Atheris (Python fuzzing)", - "Coverage-guided fuzzing for Python parsers.", - "Fuzz the Python parser entry with Atheris; assert on uncaught exceptions and resource blowups.", - languages=("python",), - tags=("parser", "fuzzable"), - vulnerability_classes=("parsing",), - stages=("hunt", "verify"), - ), - SkillDescriptor( - "property-based-testing", - "Property-based testing", - "Encode parser invariants as properties.", - "Encode parser invariants as Hypothesis properties; shrink counterexamples.", - languages=("python",), - tags=("parser",), - vulnerability_classes=("parsing",), - stages=("verify",), - ), - SkillDescriptor( - "constant-time-analysis", - "Constant-time analysis", - "Find secret-dependent timing in crypto code.", - "Find secret-dependent branches/indexing/early-exit comparisons; require constant-time primitives.", - tags=("crypto",), - vulnerability_classes=("crypto",), - stages=("hunt", "verify"), - ), - SkillDescriptor( - "wycheproof", - "Wycheproof vectors", - "Stress crypto usage with known-edge test vectors.", - "Run Wycheproof test vectors against the crypto usage; flag edge-case acceptance.", - tags=("crypto",), - vulnerability_classes=("crypto",), - stages=("verify",), - ), - SkillDescriptor( - "vector-forge", - "Crypto vector forging", - "Forge boundary vectors to probe crypto APIs.", - "Forge boundary test vectors (zero/identity/malleable signatures) to probe the crypto API.", - tags=("crypto",), - vulnerability_classes=("crypto",), - stages=("verify",), - ), - SkillDescriptor( - "zeroize-audit", - "Secret zeroization audit", - "Verify secrets are wiped after use.", - "Verify secrets are zeroized on drop; check for lingering copies.", - languages=("rust",), - tags=("crypto",), - vulnerability_classes=("crypto",), - stages=("verify",), - ), - SkillDescriptor( - "cargo-fuzz", - "cargo-fuzz", - "Coverage-guided fuzzing for Rust entries.", - "Add a cargo-fuzz target over the entry; run under ASan.", - languages=("rust",), - tags=("fuzzable", "parser"), - vulnerability_classes=("memory-safety", "parsing"), - stages=("hunt", "verify"), - ), - SkillDescriptor( - "sarif-parsing", - "SARIF parsing", - "Normalize external tool output into the map.", - "Normalize SARIF locations/rules/fingerprints into FileTarget static hints.", - vulnerability_classes=(), - stages=("map",), - ), - SkillDescriptor( - "semgrep", - "Semgrep mapping", - "Pattern sinks and iterate variants.", - "Pattern the sink shape; iterate variants; export normalized findings.", - vulnerability_classes=("injection", "deserialization"), - stages=("map",), - ), - SkillDescriptor( - "semgrep-rule-creator", - "Semgrep rule creator", - "Author a Semgrep rule for a sink.", - "Author a Semgrep rule for the sink; validate against the fixture.", - tags=("deserialization",), - vulnerability_classes=("injection", "deserialization"), - stages=("map",), - ), - SkillDescriptor( - "semgrep-rule-variant-creator", - "Semgrep variant creator", - "Widen sink coverage with variants.", - "Generate sink variants (aliases, wrappers) to widen coverage without false positives.", - vulnerability_classes=("injection",), - stages=("map",), - ), - SkillDescriptor( - "codeql", - "CodeQL dataflow", - "Model source→sink taint paths.", - "Model source→sink taint with a CodeQL query; inspect the dataflow path.", - vulnerability_classes=("injection",), - stages=("map",), - ), - SkillDescriptor( - "differential-review", - "Differential review", - "Scope the changed attack surface in a diff.", - "Scope the changed attack surface and new trust-boundary crossings in the diff.", - stages=("map", "fix"), - ), - SkillDescriptor( - "graph-evolution", - "Graph evolution", - "Bound a patch's blast radius.", - "Trace how the call/dataflow graph changed; bound the patch blast radius.", - stages=("fix",), - ), - SkillDescriptor( - "sharp-edges", - "Sharp edges review", - "Map misuse-prone APIs at the boundary.", - "Map misuse-prone APIs, insecure defaults, and confusing configuration at the boundary.", - tags=("auth_boundary", "network_entry"), - vulnerability_classes=("access-control", "api-misuse"), - stages=("map", "hunt"), - ), - SkillDescriptor( - "insecure-defaults", - "Insecure defaults review", - "Flag unsafe defaults exposed at entries.", - "Flag unsafe default flags/permissions/timeouts exposed at the entry.", - tags=("auth_boundary", "network_entry"), - vulnerability_classes=("access-control", "api-misuse"), - stages=("map", "hunt"), - ), -) - -_BY_ID = {skill.id: skill for skill in SKILL_INVENTORY} - - -@dataclass(frozen=True) -class SkillMatch: - skill: SkillDescriptor - score: int - reasons: list[str] = field(default_factory=list) - - -def route_skills( - *, - language: str | None = None, - tags: tuple[str, ...] = (), - vulnerability_classes: tuple[str, ...] = (), - stage: str | None = None, - max_snippets: int = 4, -) -> list[SkillDescriptor]: - """Select the most relevant skills for the given routing signals. - - A skill is excluded when its language list is non-empty and ``language`` is not in - it, or its stage list is non-empty and ``stage`` is not in it. Eligibility requires - at least one topical signal (tag/vulnerability-class/stage) — a pure language match - never selects a skill. Ranked by score, then id, for determinism. - """ - tag_set, vuln_set = set(tags), set(vulnerability_classes) - matches: list[SkillMatch] = [] - for skill in SKILL_INVENTORY: - if skill.languages and language is not None and language not in skill.languages: - continue - if skill.stages and stage is not None and stage not in skill.stages: - continue - signal = 2 * len(vuln_set & set(skill.vulnerability_classes)) + len(tag_set & set(skill.tags)) - if stage is not None and stage in skill.stages: - signal += 1 - if signal == 0: - continue # no topical match — do not select on language alone - rank = signal + (3 if skill.languages and language in skill.languages else 0) - matches.append(SkillMatch(skill=skill, score=rank)) - matches.sort(key=lambda match: (-match.score, match.skill.id)) - return [match.skill for match in matches[:max_snippets]] - - -def _vuln_classes_for_tags(tags: tuple[str, ...]) -> tuple[str, ...]: - return tuple(sorted({TAG_TO_VULN[tag] for tag in tags if tag in TAG_TO_VULN})) - - -def route_skills_for_target(target: FileTarget, *, stage: str | None = None, max_snippets: int = 4) -> list[SkillDescriptor]: - """Route skills for a scan target (language + tags + derived vulnerability classes).""" - tags = tuple(target.tags) - return route_skills( - language=target.language, - tags=tags, - vulnerability_classes=_vuln_classes_for_tags(tags), - stage=stage, - max_snippets=max_snippets, - ) - - -def route_skill_ids(target: FileTarget, *, stage: str | None = None, max_snippets: int = 4) -> list[str]: - return [skill.id for skill in route_skills_for_target(target, stage=stage, max_snippets=max_snippets)] - - -def select_skill_context(target: FileTarget, *, stage: str | None = None, budget_chars: int, max_snippets: int = 4) -> str: - """Pack routed skill checklists into a prompt context capped at ``budget_chars``.""" - if budget_chars <= 0: - return "" - parts: list[str] = [] - used = 0 - for skill in route_skills_for_target(target, stage=stage, max_snippets=max_snippets): - snippet = f"### {skill.title}\n{skill.checklist}" - if not parts and len(snippet) > budget_chars: - return snippet[:budget_chars] # first snippet alone exceeds the cap → truncate - if used + len(snippet) + (1 if parts else 0) > budget_chars: - break - used += len(snippet) + (1 if parts else 0) - parts.append(snippet) - return "\n".join(parts) - - -def skill_chunks(skills: tuple[SkillDescriptor, ...] | None = None, *, max_lines: int = 40) -> list[CodeChunk]: - """Chunk skill operating guidance into the ``skills`` retrieval namespace.""" - chunks: list[CodeChunk] = [] - for skill in skills or SKILL_INVENTORY: - text = f"# {skill.title}\n{skill.summary}\n\nChecklist:\n{skill.checklist}" - chunks.extend( - chunk_text_namespace( - namespace="skills", - path=f"skills/{skill.id}.md", - text=text, - metadata={ - "skill_id": skill.id, - "languages": ",".join(skill.languages), - "tags": ",".join(skill.tags), - "vulnerability_classes": ",".join(skill.vulnerability_classes), - "stages": ",".join(skill.stages), - }, - max_lines=max_lines, - ) - ) - return chunks - - -__all__ = [ - "SKILL_INVENTORY", - "STAGES", - "TAG_TO_VULN", - "SkillDescriptor", - "route_skill_ids", - "route_skills", - "route_skills_for_target", - "select_skill_context", - "skill_chunks", -] diff --git a/tests/test_agent_skills.py b/tests/test_agent_skills.py index c6a4f3d..dc45dd0 100644 --- a/tests/test_agent_skills.py +++ b/tests/test_agent_skills.py @@ -41,7 +41,7 @@ def test_scan_skill_states_the_evidence_ladder_as_the_code_defines_it() -> None: "--mode quick|standard|deep", "--fail-on", "cpg_unavailable", - "sandbox_unavailable", + "regress_retired", "report.sarif", "harness.json", "ousast mcp", diff --git a/tests/test_complexity_ledger.py b/tests/test_complexity_ledger.py index 36ac18d..8ae6c26 100644 --- a/tests/test_complexity_ledger.py +++ b/tests/test_complexity_ledger.py @@ -1,6 +1,5 @@ import json from pathlib import Path -from types import SimpleNamespace from openultrasast.complexity.ledger import ( NOT_TRIGGERABLE_DELTA, @@ -9,15 +8,11 @@ from openultrasast.complexity.ledger import ( apply_overlay, hotspot_key, load_ledger, - must_keep_as_candidate, - persist_verdicts, record_verdict, - select_forced_candidates, write_ledger, ) from openultrasast.complexity.map import Hotspot, build_complexity_map from openultrasast.findings import StaticFinding -from openultrasast.policy import CwePolicy from openultrasast.preprocess import FileTarget @@ -129,7 +124,7 @@ def test_not_triggerable_overlay_drops_score_and_keeps_inventory_hit() -> None: assert demoted.function_name == hotspot.function_name -def test_not_triggerable_overlay_drops_map_score_and_sev5_reachable_stays_candidate(tmp_path: Path) -> None: +def test_not_triggerable_overlay_drops_map_score_and_keeps_inventory(tmp_path: Path) -> None: auth = _target( "src/auth.py", loc=80, @@ -168,49 +163,6 @@ def test_not_triggerable_overlay_drops_map_score_and_sev5_reachable_stays_candid assert sev5.severity == "critical" assert sev5.rationale == ("Static pattern cmd (CWE-78) matched os.system on line 12; manual verification still required.") - policy = { - sev5.finding_id: 5, - noise.finding_id: 2, - } - forced = select_forced_candidates([sev5, noise], policy) - assert forced == (sev5,) - assert must_keep_as_candidate(sev5, 5, "reachable") is True - assert must_keep_as_candidate(noise, 2, "unknown") is False - - -def test_must_keep_as_candidate_requires_sev5_and_reachable_surface() -> None: - reachable = _hit("a.py", "a:1", reachability="reachable") - inferred = _hit("b.py", "b:1", reachability="inferred-file-surface") - unknown = _hit("c.py", "c:1", reachability="unknown") - - assert must_keep_as_candidate(reachable, 5, "reachable") is True - assert must_keep_as_candidate(inferred, 5, "inferred-file-surface") is True - assert must_keep_as_candidate(unknown, 5, "unknown") is False - assert must_keep_as_candidate(reachable, 4, "reachable") is False - assert must_keep_as_candidate(reachable, 5, None) is True - - -def test_select_forced_candidates_resolves_policy_severity() -> None: - sev5 = _hit("src/auth.py", "cmd:src/auth.py:12", reachability="reachable") - inferred = _hit("src/upload.py", "sql:src/upload.py:4", reachability="inferred-file-surface") - mild = _hit("src/helper.py", "print:src/helper.py:1", reachability="reachable") - policy = { - "CWE-78": CwePolicy("Command Injection", 5, True, False), - "CWE-89": CwePolicy("SQL Injection", 5, True, False), - "CWE-116": CwePolicy("Encode Output", 2, True, False), - } - - forced = select_forced_candidates( - [mild, sev5, inferred], - policy, - rule_cwe={ - sev5.finding_id: "CWE-78", - inferred.finding_id: "CWE-89", - mild.finding_id: "CWE-116", - }, - ) - - assert forced == (sev5, inferred) def test_ledger_round_trip_matches_design_schema(tmp_path: Path) -> None: @@ -250,25 +202,6 @@ def test_overlay_reorders_and_rebands_without_dropping_hits() -> None: assert overlaid[0].inventory_finding_ids == ("low:1",) -def test_persist_verdicts_records_triggerable_and_skips_inconclusive(tmp_path: Path) -> None: - path = tmp_path / ".openultrasast" / "calibration" / "complexity_ledger.json" - persist_verdicts( - path, - [ - SimpleNamespace(path="app.py", function_name="admin", verdict="triggerable"), - SimpleNamespace(path="safe.py", function_name=None, verdict="not_triggerable"), - SimpleNamespace(path="other.py", function_name="skip", verdict="inconclusive"), - ], - ) - loaded = load_ledger(path) - - assert loaded[hotspot_key("app.py", "admin")].last_verdict == "triggerable" - assert loaded[hotspot_key("app.py", "admin")].score_delta == TRIGGERABLE_DELTA - assert loaded[hotspot_key("safe.py", None)].last_verdict == "not_triggerable" - assert loaded[hotspot_key("safe.py", None)].score_delta == NOT_TRIGGERABLE_DELTA - assert hotspot_key("other.py", "skip") not in loaded - - def test_build_complexity_map_without_ledger_path_skips_overlay(tmp_path: Path) -> None: target = _target("src/app.py", loc=20) finding = _hit("src/app.py", "rule:src/app.py:1") diff --git a/tests/test_config.py b/tests/test_config.py index 3dfacb2..3daa2ac 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -147,26 +147,22 @@ def test_write_resolved_config_creates_json(tmp_path: Path) -> None: assert not {"dynamic", "evidence", "variants", "embeddings"} & set(payload) and "patcher" not in payload["models"] -def test_complexity_and_regress_defaults_when_section_absent() -> None: +def test_complexity_defaults_when_section_absent() -> None: config = load_config(None) assert config.complexity.max_hunter_hotspots == 8 - assert config.regress.max_candidates == 5 - assert config.regress.images == () assert config.sandbox.memory_mb == 2048 assert config.sandbox.timeout_seconds == 300 assert config.sandbox.pids_limit == 512 -def test_override_changes_max_hunter_hotspots_and_max_candidates_only(tmp_path: Path) -> None: +def test_override_changes_max_hunter_hotspots_only(tmp_path: Path) -> None: config_path = tmp_path / "openultrasast.toml" config_path.write_text( "\n".join( [ "[complexity]", "max_hunter_hotspots = 3", - "[regress]", - "max_candidates = 2", ] ) ) @@ -174,35 +170,11 @@ def test_override_changes_max_hunter_hotspots_and_max_candidates_only(tmp_path: config = load_config(config_path) assert config.complexity.max_hunter_hotspots == 3 - assert config.regress.max_candidates == 2 - assert config.regress.images == () assert config.sandbox.memory_mb == 2048 assert config.sandbox.timeout_seconds == 300 assert config.sandbox.pids_limit == 512 -def test_load_config_reads_regress_image_pins(tmp_path: Path) -> None: - config_path = tmp_path / "openultrasast.toml" - config_path.write_text( - "\n".join( - [ - "[regress]", - "max_candidates = 4", - "[regress.images]", - 'python = "python:3.12-slim"', - 'javascript = "node:22-slim"', - ] - ) - ) - - config = load_config(config_path) - - assert config.regress.max_candidates == 4 - assert dict(config.regress.images) == {"python": "python:3.12-slim", "javascript": "node:22-slim"} - assert config.complexity.max_hunter_hotspots == 8 - assert config.sandbox.memory_mb == 2048 - - def test_load_dotenv_sets_missing_keys_only(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] env_file = tmp_path / ".env" env_file.write_text("OPENROUTER_API_KEY=from-file\nS3_REGION=us-east-1\n") diff --git a/tests/test_cpg_queries_compile.py b/tests/test_cpg_queries_compile.py index b6d159c..02fcbf6 100644 --- a/tests/test_cpg_queries_compile.py +++ b/tests/test_cpg_queries_compile.py @@ -29,7 +29,7 @@ import pytest QUERIES = Path("src/openultrasast/cpg/queries") -@pytest.mark.parametrize("name", ["taint", "dominance", "config", "census", "overlay", "vocabulary"]) +@pytest.mark.parametrize("name", ["taint", "dominance", "config", "census", "overlay"]) def test_query_scope_checks_guard_nullable_methods(name: str) -> None: """PHP class constants can have no method; one unsafe scope check aborts the batch.""" script = QUERIES / f"{name}.sc" diff --git a/tests/test_degradation_matrix.py b/tests/test_degradation_matrix.py index ad94422..2ab3b2a 100644 --- a/tests/test_degradation_matrix.py +++ b/tests/test_degradation_matrix.py @@ -64,16 +64,14 @@ def test_without_the_semantic_extra_an_unsupported_language_is_named_not_guessed assert ir.reason == "language_unsupported" and ir.parse_ok is False -def test_without_a_sandbox_the_regress_stage_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A claim that could not be proved because nothing could run it is not the same as a claim that failed.""" +def test_retired_regress_stage_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: import json from openultrasast.cli import main - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") (tmp_path / "app.py").write_text("import os\n\n\ndef run(cmd):\n os.system(cmd)\n") assert main(["scan", str(tmp_path), "--mode", "deep"]) in {0, 1} runs = sorted((tmp_path / ".openultrasast" / "runs").iterdir()) manifest = json.loads((runs[-1] / "manifest.json").read_text()) reasons = {item["reason"] for item in manifest.get("degradations", [])} - assert "sandbox_unavailable" in reasons + assert "regress_retired" in reasons diff --git a/tests/test_gate.py b/tests/test_gate.py index db223f1..ac1525c 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -133,7 +133,6 @@ GOVERNANCE_SCORING_BENCHMARK_PLANES = ( "openultrasast.ruleset", "openultrasast.ruleset.store", "openultrasast.fusion", - "openultrasast.skills", "openultrasast.redaction", ) diff --git a/tests/test_hunter.py b/tests/test_hunter.py index 14e45b7..2de2b1e 100644 --- a/tests/test_hunter.py +++ b/tests/test_hunter.py @@ -2,7 +2,7 @@ import json from pathlib import Path from openultrasast.cli import main -from openultrasast.hunter import assign_tier, run_hunter_pool, schedule_hunter_tasks, select_skill_snippets, write_hunter_trajectories +from openultrasast.hunter import assign_tier, run_hunter_pool, schedule_hunter_tasks, write_hunter_trajectories from openultrasast.preprocess import FileTarget from openultrasast.rank import RankingScore @@ -13,7 +13,7 @@ def test_assign_tier_uses_priority_bands() -> None: assert assign_tier(1.99) == "C" -def test_schedule_hunter_tasks_assigns_budget_and_skill_metadata() -> None: +def test_schedule_hunter_tasks_assigns_budget() -> None: target = FileTarget( path="parser.c", absolute_path="/repo/parser.c", @@ -39,8 +39,6 @@ def test_schedule_hunter_tasks_assigns_budget_and_skill_metadata() -> None: assert tasks[0].tier == "A" assert tasks[0].budget.max_findings_per_target == 20 assert len(tasks[0].retrieval_context) == tasks[0].budget.retrieval_budget_chars - assert "address-sanitizer" in tasks[0].selected_skills - assert "harness-writing" in tasks[0].selected_skills def test_run_hunter_pool_persists_trajectory_linked_to_finding(tmp_path: Path) -> None: @@ -78,22 +76,6 @@ def test_run_hunter_pool_persists_trajectory_linked_to_finding(tmp_path: Path) - assert line["status"] == "completed" -def test_select_skill_snippets_routes_crypto_and_auth_targets() -> None: - target = FileTarget( - path="auth/crypto.py", - absolute_path="/repo/auth/crypto.py", - language="python", - loc=20, - tags=["crypto", "auth_boundary"], - has_fuzz_entry_point=False, - ) - - snippets = select_skill_snippets(target) - - assert "constant-time-analysis" in snippets - assert "sharp-edges" in snippets - - def test_standard_mode_writes_hunter_trajectory_and_manifest_ref(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] repo = tmp_path / "repo" repo.mkdir() diff --git a/tests/test_index.py b/tests/test_index.py deleted file mode 100644 index a2179ee..0000000 --- a/tests/test_index.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest - -from openultrasast.index import chunk_text_namespace - - -def test_namespace_chunks_are_metadata_scoped() -> None: - chunks = chunk_text_namespace( - namespace="skills", - path="semgrep.md", - text="Use Semgrep for variant mapping.\nKeep provenance.", - metadata={"language": "markdown", "vulnerability_class": "mapping"}, - max_lines=1, - ) - - assert len(chunks) == 2 - assert chunks[0].namespace == "skills" - assert chunks[0].language == "markdown" - assert chunks[0].metadata["vulnerability_class"] == "mapping" - assert (chunks[0].start_line, chunks[0].end_line) == (1, 1) - assert chunks[0].chunk_id != chunks[1].chunk_id - - -def test_blank_blocks_are_skipped_and_the_language_defaults_to_text() -> None: - chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n\n\nbeta\n", max_lines=1) - - assert [chunk.text for chunk in chunks] == ["alpha", "beta"] - assert chunks[0].language == "text" - - -@pytest.mark.parametrize(("namespace", "max_lines"), [("unknown", 80), ("docs", 0)]) -def test_invalid_namespace_or_window_is_rejected(namespace: str, max_lines: int) -> None: - with pytest.raises(ValueError): - chunk_text_namespace(namespace=namespace, path="x", text="y", max_lines=max_lines) diff --git a/tests/test_module_audit.py b/tests/test_module_audit.py deleted file mode 100644 index a860ce1..0000000 --- a/tests/test_module_audit.py +++ /dev/null @@ -1,93 +0,0 @@ -"""model-grounded-detection Req 3: outgrowth is visible and checkable, not noticed two specs later.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -SRC = Path("src/openultrasast") -MANIFEST = Path("benchmarks/measurements/2026-09-08-module-audit.json") - - -def test_no_module_in_the_tree_is_orphaned() -> None: - from openultrasast.model.audit import audit - - orphans = [row.module for row in audit(SRC) if row.classification == "orphaned"] - assert not orphans, f"orphaned modules must be removed, not carried: {orphans}" - - -def test_every_module_is_classified() -> None: - from openultrasast.model.audit import audit, module_name - - classified = {row.module for row in audit(SRC)} - on_disk = {module_name(path, SRC) for path in SRC.rglob("*.py") if "__pycache__" not in path.parts} - assert on_disk - classified <= {""}, f"unclassified: {sorted(on_disk - classified - {''})}" - - -def test_the_named_subsystems_each_carry_an_explicit_decision() -> None: - """Req 3.3: the older-phase subsystems still in the tree get a decision, not silence.""" - from openultrasast.model.audit import audit - - rows = {row.module: row for row in audit(SRC)} - for name in ("fusion", "mcp", "skills"): - assert name in rows, f"{name} vanished without a recorded decision" - assert rows[name].reason, f"{name} has no stated reason" - # The removed agentic plane's modules are gone from the tree, not carried as rows (harnessx-removal Req 2.1). - assert not {"harness_ext", "hunter_harness", "verify_judge"} & set(rows) - # The five the reachability-first audit found orphaned are gone too (legacy cleanup 2026-10-02). - assert not {"model.judge", "model.execution", "model.calibrate", "stage_processors", "slot_contract"} & set(rows) - - -def test_the_manifest_matches_the_tree() -> None: - """A later reader can check the tree against the committed manifest (Req 3.4).""" - from openultrasast.model.audit import audit, to_dict - - assert MANIFEST.is_file(), "the audit manifest is committed evidence, not a local artifact" - committed = json.loads(MANIFEST.read_text()) - current = to_dict(audit(SRC)) - assert committed["counts"] == current["counts"], "the tree drifted from its committed audit" - assert committed["orphaned"] == current["orphaned"] == [] - - -def _tree(root: Path, files: dict[str, str]) -> Path: - for name, text in files.items(): - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text) - return root - - -def test_a_spec_listed_module_nothing_imports_is_still_orphaned(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Reachability decides; SPEC_OWNED and STANDALONE only supply the reason (the list hid five dead modules).""" - from openultrasast.model import audit as module - - src = _tree(tmp_path, {"cli.py": "from . import used\n", "used.py": "", "ghost.py": "", "lonely.py": ""}) - monkeypatch.setitem(module.SPEC_OWNED, "ghost", "some spec Req 1: still listed") - monkeypatch.setitem(module.STANDALONE, "lonely", "its own entry point, supposedly") - rows = {row.module: row for row in module.audit(src)} - assert rows["used"].classification == "load_bearing" - assert rows["ghost"].classification == "orphaned" - assert "still listed" in rows["ghost"].reason - assert rows["lonely"].classification == "orphaned" - - -def test_submodule_imports_and_python_m_modules_count_as_reachable(tmp_path: Path) -> None: - from openultrasast.model.audit import audit - - src = _tree( - tmp_path, - { - "cli.py": "def run():\n from .pkg import sub\n", - "pkg/__init__.py": "", - "pkg/sub.py": "", - "tool.py": 'from . import helper\n\nif __name__ == "__main__":\n pass\n', - "helper.py": "", - "dead.py": "from . import deader\n", - "deader.py": "", - }, - ) - rows = {row.module: row.classification for row in audit(src)} - assert rows["pkg.sub"] == rows["tool"] == rows["helper"] == "load_bearing" - assert rows["dead"] == rows["deader"] == "orphaned" diff --git a/tests/test_obligation_scan.py b/tests/test_obligation_scan.py index 2a9e214..8a8bd0f 100644 --- a/tests/test_obligation_scan.py +++ b/tests/test_obligation_scan.py @@ -92,19 +92,3 @@ def test_quick_scan_and_disabled_config_skip_obligations(tmp_path: Path, monkeyp assert not any(f["finding_id"].startswith("obligation:") for f in findings) -def test_deep_scan_never_offers_obligation_findings_to_the_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast import cli - from openultrasast.sandbox import FakeSandboxRunner, SandboxResult - - repo = _repo(tmp_path, monkeypatch) - (repo / "openultrasast.toml").write_text("[obligations]\nmin_siblings = 2\n") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False)) - monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake) - assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0 - run_dir = _run_dir(repo) - verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] - offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]} - assert not any(finding_id.startswith("obligation:") for finding_id in offered) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - assert any(f["finding_id"].startswith("obligation:") for f in findings) # still reported diff --git a/tests/test_regress_candidates.py b/tests/test_regress_candidates.py deleted file mode 100644 index 0cb9fb5..0000000 --- a/tests/test_regress_candidates.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Bounded regression candidate selection from hotspots and severe inventory (task 5.1).""" - -from openultrasast.complexity.ledger import NOT_TRIGGERABLE_DELTA, apply_overlay, record_verdict -from openultrasast.complexity.map import Hotspot -from openultrasast.findings import StaticFinding -from openultrasast.regress.candidate import select_candidates - - -def _finding( - path: str, - finding_id: str, - *, - function_name: str | None = None, - reachability: str = "unknown", - severity: str = "low", -) -> StaticFinding: - return StaticFinding( - finding_id=finding_id, - path=path, - title="pattern hit", - severity=severity, - confidence="medium", - evidence_level="static_corroboration", - rationale="inventory", - line=1, - function_name=function_name, - reachability_status=reachability, - reachability_evidence=[], - reachability_conditions=[], - tags=[], - ranking_priority=1.0, - ) - - -def _hotspot( - path: str, - *, - function_name: str | None = None, - score: float = 4.0, - band: str = "medium", - inventory_finding_ids: tuple[str, ...] = (), -) -> Hotspot: - return Hotspot( - path=path, - function_name=function_name, - score=score, - band=band, - signals={"loc": 10, "inventory_hit_count": len(inventory_finding_ids)}, - rationale=f"{band} band: score {score}", - test_hint=None, - inventory_finding_ids=inventory_finding_ids, - ) - - -def _six_hotspots(*, sev5_id: str | None = None, sev5_score: float = 4.0) -> tuple[Hotspot, ...]: - return ( - _hotspot("a.py", function_name="a", score=5.0), - _hotspot("b.py", function_name="b", score=4.8), - _hotspot("c.py", function_name="c", score=4.6), - _hotspot("d.py", function_name="d", score=4.4), - _hotspot("e.py", function_name="e", score=4.2), - _hotspot( - "f.py", - function_name="f", - score=sev5_score, - inventory_finding_ids=(sev5_id,) if sev5_id else (), - ), - ) - - -def test_six_hotspots_max_candidates_three_returns_three() -> None: - noise = _finding("f.py", "print:f.py:1", function_name="f", reachability="reachable") - selected = select_candidates( - _six_hotspots(), - [noise], - max_candidates=3, - policy_severity_by_id={noise.finding_id: 2}, - ) - - assert len(selected) == 3 - assert [item.path for item in selected] == ["a.py", "b.py", "c.py"] - - -def test_sev5_reachable_extra_included_when_ledger_demoted_hotspot() -> None: - sev5_id = "cmd:f.py:1" - hotspots = _six_hotspots(sev5_id=sev5_id, sev5_score=5.2) - ledger = record_verdict({}, path="f.py", function_name="f", verdict="not_triggerable") - demoted = apply_overlay(hotspots, ledger) - finding = _finding( - "f.py", - sev5_id, - function_name="f", - reachability="reachable", - severity="critical", - ) - - selected = select_candidates( - demoted, - [finding], - max_candidates=3, - policy_severity_by_id={sev5_id: 5}, - ) - - demoted_f = next(item for item in demoted if item.path == "f.py") - assert demoted_f.score == round(5.2 + NOT_TRIGGERABLE_DELTA, 4) - assert [item.path for item in demoted[:3]] == ["a.py", "b.py", "c.py"] - assert demoted_f not in demoted[:3] - assert len(selected) == 4 - assert [item.path for item in selected[:3]] == ["a.py", "b.py", "c.py"] - assert selected[3] is demoted_f - assert sev5_id in selected[3].inventory_finding_ids - - -def test_sev5_already_in_cap_is_not_duplicated() -> None: - sev5_id = "cmd:a.py:1" - hotspots = ( - _hotspot("a.py", function_name="a", score=5.0, inventory_finding_ids=(sev5_id,)), - _hotspot("b.py", function_name="b", score=4.8), - _hotspot("c.py", function_name="c", score=4.6), - _hotspot("d.py", function_name="d", score=4.4), - _hotspot("e.py", function_name="e", score=4.2), - _hotspot("f.py", function_name="f", score=4.0), - ) - finding = _finding("a.py", sev5_id, function_name="a", reachability="reachable", severity="critical") - - selected = select_candidates( - hotspots, - [finding], - max_candidates=3, - policy_severity_by_id={sev5_id: 5}, - ) - - assert len(selected) == 3 - assert [item.path for item in selected] == ["a.py", "b.py", "c.py"] - assert sev5_id in selected[0].inventory_finding_ids - - -def test_sev5_reachable_inventory_without_hotspot_is_still_selected() -> None: - finding = _finding( - "z.py", - "cmd:z.py:1", - function_name="z", - reachability="reachable", - severity="critical", - ) - - selected = select_candidates( - _six_hotspots(), - [finding], - max_candidates=3, - policy_severity_by_id={finding.finding_id: 5}, - ) - - assert len(selected) == 4 - assert [item.path for item in selected[:3]] == ["a.py", "b.py", "c.py"] - extra = selected[3] - assert extra.path == "z.py" - assert extra.function_name == "z" - assert extra.inventory_finding_ids == (finding.finding_id,) diff --git a/tests/test_regress_recipes.py b/tests/test_regress_recipes.py deleted file mode 100644 index a2bf609..0000000 --- a/tests/test_regress_recipes.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Language recipes for isolated regression snippets (task 5.2).""" - -from pathlib import Path - -from openultrasast.config import SandboxConfig -from openultrasast.regress import INCONCLUSIVE, RegressionRunner -from openultrasast.regress.recipes import recipe_for -from openultrasast.sandbox import FakeSandboxRunner, SandboxJob - -LIMITS = SandboxConfig(memory_mb=256, timeout_seconds=30, pids_limit=128) -PYTHON_SNIPPET = "assert 1 + 1 == 2\n" -JS_SNIPPET = "console.log('ok')\n" -C_SNIPPET = "int main(void) { return 0; }\n" -JAVA_SNIPPET = "class Case { public static void main(String[] args) {} }\n" - - -def _repo(tmp_path: Path) -> Path: - repo = tmp_path / "repo" - repo.mkdir() - return repo - - -def test_python_recipe_runs_scratch_snippet_with_repo_on_pythonpath(tmp_path: Path) -> None: - repo = _repo(tmp_path) - job = recipe_for("python", PYTHON_SNIPPET, "python:3.12-alpine", LIMITS, repo_root=repo) - - assert isinstance(job, SandboxJob) - joined = " ".join(job.command) - assert job.image == "python:3.12-alpine" - assert job.repo_root == repo - assert job.scratch_files == {"case.py": PYTHON_SNIPPET} - assert "python" in job.command - assert "/scratch/case.py" in job.command - assert "PYTHONPATH=/workspace" in joined - assert job.timeout_seconds == LIMITS.timeout_seconds - assert job.memory_mb == LIMITS.memory_mb - assert job.pids_limit == LIMITS.pids_limit - - -def test_javascript_recipe_runs_scratch_snippet_with_repo_on_node_path(tmp_path: Path) -> None: - repo = _repo(tmp_path) - job = recipe_for("javascript", JS_SNIPPET, "node:22-alpine", LIMITS, repo_root=repo) - - assert isinstance(job, SandboxJob) - joined = " ".join(job.command) - assert job.image == "node:22-alpine" - assert job.repo_root == repo - assert job.scratch_files == {"case.js": JS_SNIPPET} - assert "node" in job.command - assert "/scratch/case.js" in job.command - assert "NODE_PATH=/workspace" in joined - assert job.timeout_seconds == LIMITS.timeout_seconds - assert job.memory_mb == LIMITS.memory_mb - assert job.pids_limit == LIMITS.pids_limit - - -def test_c_recipe_compiles_and_runs_when_image_has_compiler(tmp_path: Path) -> None: - repo = _repo(tmp_path) - job = recipe_for("c", C_SNIPPET, "gcc:13", LIMITS, repo_root=repo) - - assert isinstance(job, SandboxJob) - joined = " ".join(job.command) - assert job.scratch_files == {"case.c": C_SNIPPET} - assert "cc" in joined - assert "/scratch/case.c" in joined - assert "-o" in joined - assert "/scratch/case" in joined - - -def test_unknown_language_recipe_is_none(tmp_path: Path) -> None: - repo = _repo(tmp_path) - assert recipe_for("java", JAVA_SNIPPET, "eclipse-temurin:21", LIMITS, repo_root=repo) is None - assert recipe_for("unknown", "print(1)\n", "python:3.12-alpine", LIMITS, repo_root=repo) is None - - -def test_python_and_javascript_recipes_produce_runner_jobs(tmp_path: Path) -> None: - repo = _repo(tmp_path) - python_job = recipe_for("python", PYTHON_SNIPPET, "python:3.12-alpine", LIMITS, repo_root=repo) - js_job = recipe_for("javascript", JS_SNIPPET, "node:22-alpine", LIMITS, repo_root=repo) - sandbox = FakeSandboxRunner() - runner = RegressionRunner(sandbox) - - python_verdict = runner.run_recipe("python", PYTHON_SNIPPET, "python:3.12-alpine", LIMITS, repo_root=repo) - js_verdict = runner.run_recipe("javascript", JS_SNIPPET, "node:22-alpine", LIMITS, repo_root=repo) - - assert python_job is not None - assert js_job is not None - assert python_verdict.reason != "missing_recipe" - assert js_verdict.reason != "missing_recipe" - assert sandbox.jobs == [python_job, js_job] - - -def test_unknown_language_returns_inconclusive_without_calling_runner(tmp_path: Path) -> None: - repo = _repo(tmp_path) - sandbox = FakeSandboxRunner() - runner = RegressionRunner(sandbox) - - assert recipe_for("java", JAVA_SNIPPET, "eclipse-temurin:21", LIMITS, repo_root=repo) is None - verdict = runner.run_recipe("java", JAVA_SNIPPET, "eclipse-temurin:21", LIMITS, repo_root=repo) - - assert verdict.verdict == INCONCLUSIVE - assert verdict.reason == "missing_recipe" - assert sandbox.jobs == [] diff --git a/tests/test_regress_safety.py b/tests/test_regress_safety.py deleted file mode 100644 index c8360e3..0000000 --- a/tests/test_regress_safety.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Reject unsafe regression snippets before the sandbox runner sees them (task 4.3).""" - -from pathlib import Path - -import pytest - -from openultrasast.regress import RegressionRunner, UnsafeSnippetError, check_snippet_safety -from openultrasast.sandbox import FakeSandboxRunner, SandboxJob - -BENIGN_ASSERT = "assert 1 + 1 == 2\n" - - -def _job(tmp_path: Path, snippet: str) -> SandboxJob: - repo = tmp_path / "repo" - repo.mkdir() - return SandboxJob( - image="python:3.12-alpine", - command=("python", "/scratch/case.py"), - repo_root=repo, - scratch_files={"case.py": snippet}, - timeout_seconds=30, - memory_mb=256, - pids_limit=128, - ) - - -def test_docker_sock_rejected_does_not_create_runner_job(tmp_path: Path) -> None: - snippet = 'client = docker.DockerClient(base_url="unix://var/run/docker.sock")\n' - sandbox = FakeSandboxRunner() - runner = RegressionRunner(sandbox) - - with pytest.raises(UnsafeSnippetError, match="Docker socket"): - check_snippet_safety(snippet) - verdict = runner.run_snippet(snippet, _job(tmp_path, snippet)) - - assert verdict.verdict == "inconclusive" - assert verdict.reason == "safety_rejected" - assert sandbox.jobs == [] - - -def test_host_network_rejected_does_not_create_runner_job(tmp_path: Path) -> None: - snippet = "subprocess.run(['docker', 'run', '--network', 'host', 'alpine'])\n" - sandbox = FakeSandboxRunner() - runner = RegressionRunner(sandbox) - - with pytest.raises(UnsafeSnippetError, match="host network"): - check_snippet_safety(snippet) - verdict = runner.run_snippet(snippet, _job(tmp_path, snippet)) - - assert verdict.verdict == "inconclusive" - assert verdict.reason == "safety_rejected" - assert sandbox.jobs == [] - - -def test_write_under_workspace_rejected_does_not_create_runner_job(tmp_path: Path) -> None: - snippet = 'open("/workspace/pwned.py", "w").write("owned")\n' - sandbox = FakeSandboxRunner() - runner = RegressionRunner(sandbox) - - with pytest.raises(UnsafeSnippetError, match="/workspace"): - check_snippet_safety(snippet) - verdict = runner.run_snippet(snippet, _job(tmp_path, snippet)) - - assert verdict.verdict == "inconclusive" - assert verdict.reason == "safety_rejected" - assert sandbox.jobs == [] - - -def test_benign_python_assert_snippet_is_allowed(tmp_path: Path) -> None: - assert check_snippet_safety(BENIGN_ASSERT) is None - sandbox = FakeSandboxRunner() - job = _job(tmp_path, BENIGN_ASSERT) - RegressionRunner(sandbox).run_snippet(BENIGN_ASSERT, job) - assert sandbox.jobs == [job] - - -@pytest.mark.parametrize( - "snippet", - [ - "import socket\nsocket.create_connection(('127.0.0.1', 80))\n", - "curl https://example.invalid/pwn\n", - ], -) -def test_socket_and_curl_mentions_are_rejected(snippet: str) -> None: - with pytest.raises(UnsafeSnippetError): - check_snippet_safety(snippet) diff --git a/tests/test_regress_verdicts.py b/tests/test_regress_verdicts.py deleted file mode 100644 index f2e143f..0000000 --- a/tests/test_regress_verdicts.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Map sandbox outcomes to the four verdicts and the worth-fixing gate (task 5.3).""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.regress import RegressionRunner -from openultrasast.regress.verdict import ( - ALREADY_COVERED, - INCONCLUSIVE, - NOT_TRIGGERABLE, - TRIGGERABLE, - is_worth_fixing, - verdict_from_result, -) -from openultrasast.sandbox import FakeSandboxRunner, SandboxJob, SandboxResult - -_REACHABLE = "reachable" -_INFERRED = "inferred-file-surface" -_UNKNOWN = "unknown" - - -def _result(*, exit_code: int = 0, stdout: str = "", stderr: str = "", timed_out: bool = False) -> SandboxResult: - return SandboxResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=timed_out) - - -def _job(tmp_path: Path, snippet: str) -> SandboxJob: - repo = tmp_path / "repo" - repo.mkdir() - return SandboxJob( - image="python:3.12-alpine", - command=("python", "/scratch/case.py"), - repo_root=repo, - scratch_files={"case.py": snippet}, - timeout_seconds=30, - memory_mb=256, - pids_limit=128, - ) - - -# observation kwargs, sandbox result, reachability, expected verdict, expected reason, worth-fixing -_CASES = ( - pytest.param( - {"recipe_missing": True}, - None, - _REACHABLE, - INCONCLUSIVE, - "missing_recipe", - False, - id="recipe-missing-inconclusive", - ), - pytest.param( - {"sandbox_missing": True}, - None, - _REACHABLE, - INCONCLUSIVE, - "sandbox_unavailable", - False, - id="sandbox-missing-inconclusive", - ), - pytest.param( - {}, - _result(exit_code=-1, timed_out=True), - _REACHABLE, - INCONCLUSIVE, - "timeout", - False, - id="timeout-inconclusive", - ), - pytest.param( - {"safety_rejected": True}, - None, - _INFERRED, - INCONCLUSIVE, - "safety_rejected", - False, - id="safety-reject-inconclusive", - ), - pytest.param( - {"covering_test": True}, - _result(exit_code=0, stdout="ok\n"), - _REACHABLE, - ALREADY_COVERED, - "covered_test_passed", - False, - id="covering-test-passed-already-covered", - ), - pytest.param( - {}, - _result(exit_code=1, stderr="AssertionError: boom\n"), - _REACHABLE, - TRIGGERABLE, - "assertion", - True, - id="assertion-triggerable-reachable-worth-fixing", - ), - pytest.param( - {}, - _result(exit_code=1, stderr="ERROR: AddressSanitizer: heap-buffer-overflow\n"), - _INFERRED, - TRIGGERABLE, - "sanitizer_abort", - True, - id="sanitizer-triggerable-inferred-worth-fixing", - ), - pytest.param( - {}, - _result(exit_code=139, stderr="Segmentation fault\n"), - _UNKNOWN, - TRIGGERABLE, - "crash", - False, - id="crash-triggerable-unknown-not-worth-fixing", - ), - pytest.param( - {}, - _result(exit_code=0, stdout="ok\n"), - _REACHABLE, - NOT_TRIGGERABLE, - "exit_zero", - False, - id="exit-zero-not-triggerable", - ), - pytest.param( - {"covering_test": True}, - _result(exit_code=0), - _UNKNOWN, - ALREADY_COVERED, - "covered_test_passed", - False, - id="already-covered-unknown-does-not-fail-build", - ), - pytest.param( - {}, - _result(exit_code=0), - _INFERRED, - NOT_TRIGGERABLE, - "exit_zero", - False, - id="not-triggerable-inferred-does-not-fail-build", - ), - pytest.param( - {}, - _result(exit_code=-1, timed_out=True), - _INFERRED, - INCONCLUSIVE, - "timeout", - False, - id="inconclusive-inferred-does-not-fail-build", - ), -) - - -@pytest.mark.parametrize( - ("flags", "result", "reachability", "expected_verdict", "expected_reason", "expected_worth_fixing"), - _CASES, -) -def test_sandbox_outcomes_map_to_verdicts_and_worth_fixing( - flags: dict[str, bool], - result: SandboxResult | None, - reachability: str, - expected_verdict: str, - expected_reason: str, - expected_worth_fixing: bool, -) -> None: - mapped = verdict_from_result(result, **flags) - - assert mapped.verdict == expected_verdict - assert mapped.reason == expected_reason - assert mapped.reason - assert is_worth_fixing(mapped.verdict, reachability) is expected_worth_fixing - - -def test_table_covers_four_verdicts_and_worth_fixing_boolean() -> None: - verdicts = {case.values[3] for case in _CASES} - worth_fixing_values = {case.values[5] for case in _CASES} - - assert verdicts == {TRIGGERABLE, NOT_TRIGGERABLE, ALREADY_COVERED, INCONCLUSIVE} - assert worth_fixing_values == {True, False} - - -def test_runner_maps_timeout_crash_and_exit_zero(tmp_path: Path) -> None: - snippet = "assert 1 + 1 == 2\n" - job = _job(tmp_path, snippet) - timeout = FakeSandboxRunner(_result(exit_code=-1, timed_out=True)) - crash = FakeSandboxRunner(_result(exit_code=1, stderr="AssertionError\n")) - ok = FakeSandboxRunner(_result(exit_code=0)) - - timed_out = RegressionRunner(timeout).run_snippet(snippet, job) - triggered = RegressionRunner(crash).run_snippet(snippet, job) - not_triggered = RegressionRunner(ok).run_snippet(snippet, job) - - assert timed_out.verdict == INCONCLUSIVE - assert timed_out.reason == "timeout" - assert triggered.verdict == TRIGGERABLE - assert not_triggered.verdict == NOT_TRIGGERABLE - assert is_worth_fixing(timed_out.verdict, _REACHABLE) is False - assert is_worth_fixing(triggered.verdict, _REACHABLE) is True - assert is_worth_fixing(not_triggered.verdict, _REACHABLE) is False diff --git a/tests/test_reports.py b/tests/test_reports.py index e6b7fcd..17d02c8 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,7 +1,6 @@ import json import subprocess from pathlib import Path -from types import SimpleNamespace import pytest @@ -9,7 +8,6 @@ from openultrasast.cli import main from openultrasast.complexity.hints import TestHint from openultrasast.complexity.map import ComplexityMap, Hotspot from openultrasast.findings import StaticFinding -from openultrasast.regress import CandidateVerdict from openultrasast.reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report from openultrasast.run import ScanRun from openultrasast.stages import Stage, plan_for_mode, record_completed, stages_payload @@ -99,7 +97,7 @@ def test_markdown_report_includes_verification_status(tmp_path: Path) -> None: assert "worth_fixing" not in text.lower() -def test_markdown_report_includes_complexity_map_and_worth_fixing_sections(tmp_path: Path) -> None: +def test_markdown_report_includes_complexity_map(tmp_path: Path) -> None: finding = _finding() output = tmp_path / "report.md" hint = TestHint( @@ -119,47 +117,21 @@ def test_markdown_report_includes_complexity_map_and_worth_fixing_sections(tmp_p test_hint=hint, inventory_finding_ids=(finding.finding_id,), ) - verdict = CandidateVerdict( - path="app.py", - function_name="admin", - language="python", - verdict="triggerable", - reason="crash", - inventory_finding_ids=(finding.finding_id,), - worth_fixing=True, - ) - skipped = CandidateVerdict( - path="safe.py", - function_name=None, - language="python", - verdict="not_triggerable", - reason="exit_zero", - inventory_finding_ids=(), - worth_fixing=False, - ) - write_markdown_report( [finding], output, [verify_finding(finding)], complexity_map=ComplexityMap(hotspots=(hotspot,), heuristic_only=True), - verdicts=(verdict, skipped), ) text = output.read_text() inventory_at = text.index("## Inventory") map_at = text.index("## Complexity map") - worth_at = text.index("## Worth fixing") - assert inventory_at < map_at < worth_at - map_section = text[map_at:worth_at] - worth_section = text[worth_at:] + assert inventory_at < map_at + map_section = text[map_at:] assert "app.py::admin" in map_section assert "Gap: `no_adjacent_test`" in map_section assert "Recommended test: `http-contract`" in map_section - assert "triggerable" in worth_section - assert "crash" in worth_section - assert "safe.py" not in worth_section - assert "## Inventory" in text def test_markdown_report_labels_hits_as_inventory_not_worth_fixing(tmp_path: Path) -> None: @@ -178,30 +150,12 @@ def test_markdown_report_labels_hits_as_inventory_not_worth_fixing(tmp_path: Pat def test_scan_exit_code_policy() -> None: finding = _finding() verification = verify_finding(finding) - worth_fixing = SimpleNamespace(worth_fixing=True) assert scan_exit_code([finding], [verification], "never") == 0 assert scan_exit_code([finding], [verification], "findings") == 1 assert scan_exit_code([], [], "findings") == 0 assert scan_exit_code([finding], [verification], "verified") == 1 assert scan_exit_code([], [], "verified") == 0 - # findings/verified ignore worth-fixing verdicts - assert scan_exit_code([], [], "findings", worth_fixing_verdicts=[worth_fixing]) == 0 - assert scan_exit_code([], [], "verified", worth_fixing_verdicts=[worth_fixing]) == 0 - assert scan_exit_code([finding], [verification], "findings", worth_fixing_verdicts=[worth_fixing]) == 1 - assert scan_exit_code([finding], [verification], "verified", worth_fixing_verdicts=[worth_fixing]) == 1 - - -def test_scan_exit_code_worth_fixing() -> None: - finding = _finding() - verification = verify_finding(finding) - worth_fixing = SimpleNamespace(worth_fixing=True) - not_worth_fixing = SimpleNamespace(worth_fixing=False) - - assert scan_exit_code([finding], [verification], "worth-fixing") == 0 - assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[]) == 0 - assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[not_worth_fixing]) == 0 - assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[worth_fixing]) == 1 def test_scan_exit_code_unknown_policy() -> None: @@ -209,16 +163,6 @@ def test_scan_exit_code_unknown_policy() -> None: scan_exit_code([], [], "maybe") -def test_scan_help_lists_worth_fixing(capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit) as exc: - main(["scan", "--help"]) - assert exc.value.code == 0 - help_text = capsys.readouterr().out - assert "worth-fixing" in help_text - assert "findings" in help_text - assert "verified" in help_text - - def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] repo = tmp_path / "repo" repo.mkdir() @@ -236,20 +180,9 @@ def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, m assert manifest["findings"][0]["artifact_refs"]["verification_json"] == "verification.json" -def test_cli_scan_fail_on_worth_fixing_exits_zero_without_verdicts(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n") - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - - assert main(["scan", str(repo), "--mode", "quick", "--fail-on", "worth-fixing"]) == 0 - - def test_quick_scan_report_is_inventory_without_map_or_docker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: from openultrasast import tool_hunter from openultrasast.complexity import map as complexity_map - from openultrasast.sandbox import probe as sandbox_probe - from openultrasast.sandbox import runner as sandbox_runner repo = tmp_path / "repo" repo.mkdir() @@ -273,8 +206,6 @@ def test_quick_scan_report_is_inventory_without_map_or_docker(tmp_path: Path, mo return _fail monkeypatch.setattr(subprocess, "run", guarded_run) - monkeypatch.setattr(sandbox_runner, "build_docker_argv", fail("quick scan must not build docker argv")) - monkeypatch.setattr(sandbox_probe.SandboxProbe, "available", fail("quick scan must not probe docker")) monkeypatch.setattr(complexity_map, "build_complexity_map", fail("quick scan must not build a complexity map")) monkeypatch.setattr(tool_hunter, "run_tool_hunter", fail("quick scan must not start a hunter model")) @@ -297,8 +228,6 @@ def test_quick_scan_report_is_inventory_without_map_or_docker(tmp_path: Path, mo def test_standard_scan_writes_complexity_map_without_docker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.sandbox import probe as sandbox_probe - from openultrasast.sandbox import runner as sandbox_runner repo = tmp_path / "repo" repo.mkdir() @@ -315,15 +244,7 @@ def test_standard_scan_writes_complexity_map_without_docker(tmp_path: Path, monk raise AssertionError(f"standard scan must not start docker: {argv}") return original_run(command, *args, **kwargs) - def fail(message: str): # type: ignore[no-untyped-def] - def _fail(*args, **kwargs): # type: ignore[no-untyped-def] - raise AssertionError(message) - - return _fail - monkeypatch.setattr(subprocess, "run", guarded_run) - monkeypatch.setattr(sandbox_runner, "build_docker_argv", fail("standard scan must not build docker argv")) - monkeypatch.setattr(sandbox_probe.SandboxProbe, "available", fail("standard scan must not probe docker")) assert main(["scan", str(repo), "--mode", "standard"]) == 0 @@ -351,23 +272,12 @@ def test_standard_scan_writes_complexity_map_without_docker(tmp_path: Path, monk def test_standard_scan_records_hunter_model_unavailable_when_model_unset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.sandbox import probe as sandbox_probe - from openultrasast.sandbox import runner as sandbox_runner repo = tmp_path / "repo" repo.mkdir() (repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n") monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - def fail(message: str): # type: ignore[no-untyped-def] - def _fail(*args, **kwargs): # type: ignore[no-untyped-def] - raise AssertionError(message) - - return _fail - - monkeypatch.setattr(sandbox_runner, "build_docker_argv", fail("standard scan must not build docker argv")) - monkeypatch.setattr(sandbox_probe.SandboxProbe, "available", fail("standard scan must not probe docker")) - assert main(["scan", str(repo), "--mode", "standard"]) == 0 run_dir = sorted((repo / ".runs").iterdir())[-1] @@ -408,38 +318,6 @@ def test_manifest_includes_complexity_summary(tmp_path: Path) -> None: assert payload["artifacts"]["complexity_map"] == "complexity_map.json" -def test_manifest_includes_worth_fixing_count(tmp_path: Path) -> None: - run = ScanRun(scan_id="scan-1", root=tmp_path / "run", target=tmp_path / "repo") - run.root.mkdir() - finding = _finding() - artifacts = { - "findings": run.root / "findings.json", - "verification": run.root / "verification.json", - "markdown": run.root / "report.md", - "sarif": run.root / "report.sarif", - "verdicts": run.root / "verdicts.json", - } - output = run.root / "manifest.json" - - write_manifest( - run=run, - findings=[finding], - verifications=[verify_finding(finding)], - artifact_paths=artifacts, - path=output, - stages={"requested": ["static", "map", "regress"], "completed": ["static", "map", "regress"], "skipped": []}, - worth_fixing={ - "count": 1, - "verdicts": [{"path": "app.py", "function_name": "admin", "verdict": "triggerable", "reason": "crash"}], - }, - ) - - payload = json.loads(output.read_text()) - assert payload["stages"]["completed"] == ["static", "map", "regress"] - assert payload["worth_fixing"]["count"] == 1 - assert payload["worth_fixing"]["verdicts"][0]["verdict"] == "triggerable" - - def _finding() -> StaticFinding: return StaticFinding( finding_id="python-unsafe-eval:app.py:3", @@ -669,7 +547,7 @@ def test_every_recorded_scan_degradation_is_rendered(tmp_path) -> None: # type: {"stage": "model", "reason": "model_withheld_execution_budget"}, {"stage": "budget", "reason": "max_findings_exceeded", "requested": 5, "actual": 9}, {"stage": "map", "reason": "hunter_model_unavailable"}, - {"stage": "regress", "reason": "sandbox_unavailable"}, + {"stage": "regress", "reason": "regress_retired"}, {"stage": "x", "reason": "not_yet_described"}, ] path = tmp_path / "report.md" @@ -684,7 +562,8 @@ def test_every_recorded_scan_degradation_is_rendered(tmp_path) -> None: # type: "time budget leaves no room", "first 5 of 9 findings", "needs a model and none is available", - "Docker is not available", + "the sandboxed snippet stage was retired on 2026-10-13", + "execution proof comes from the search-with-proof arm", "`not_yet_described`", ): assert needle in text, needle @@ -697,3 +576,30 @@ def test_quick_scan_names_a_language_nothing_covers(tmp_path, capsys) -> None: assert "notice: language not covered: go (1 files)" in capsys.readouterr().out report = next((tmp_path / ".openultrasast" / "runs").iterdir()) / "report.md" assert "Not a clean result for go (1 files)" in report.read_text() + + +@pytest.mark.parametrize("fail_on, expected", [("never", 0), ("findings", 1), ("verified", 1)]) +def test_deep_scan_records_retirement_without_running_snippets(tmp_path, monkeypatch, fail_on, expected): + (tmp_path / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n") + + def guard(command, *args, **kwargs): + argv = list(command) if isinstance(command, list | tuple) else [command] + assert not argv or Path(str(argv[0])).name != "docker", "deep scan must not start Docker" + return original_run(command, *args, **kwargs) + + original_run = subprocess.run + monkeypatch.setattr(subprocess, "run", guard) + assert main(["scan", str(tmp_path), "--mode", "deep", "--fail-on", fail_on]) == expected + run_dir = next((tmp_path / ".openultrasast" / "runs").iterdir()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + retired = {"stage": "regress", "reason": "regress_retired"} + assert manifest["stages"] == { + "requested": ["static", "map", "regress"], + "completed": ["static", "map"], + "skipped": [retired], + } + assert manifest["degradations"].count(retired) == 1 + assert "verdicts" not in manifest["artifacts"] + assert "worth_fixing" not in manifest + assert not (run_dir / "verdicts.json").exists() + assert "execution proof comes from the search-with-proof arm" in (run_dir / "report.md").read_text() diff --git a/tests/test_sandbox_probe.py b/tests/test_sandbox_probe.py deleted file mode 100644 index ceb7717..0000000 --- a/tests/test_sandbox_probe.py +++ /dev/null @@ -1,117 +0,0 @@ -import subprocess -from typing import Any - -import pytest - -from openultrasast.sandbox import FakeSandboxProbe, SandboxProbe, resolve_sandbox_probe -from openultrasast.sandbox import probe as probe_mod - - -def _completed(returncode: int) -> subprocess.CompletedProcess[str]: - return subprocess.CompletedProcess(["docker", "info"], returncode, stdout="", stderr="") - - -def test_fake_probe_can_be_forced_off() -> None: - probe = FakeSandboxProbe(available=False) - assert probe.available() is False - - -def test_fake_probe_can_be_forced_on() -> None: - probe = FakeSandboxProbe(available=True) - assert probe.available() is True - - -def test_resolve_probe_forced_off_via_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(probe_mod.PROBE_ENV, "0") - probe = resolve_sandbox_probe() - assert isinstance(probe, FakeSandboxProbe) - assert probe.available() is False - - -def test_resolve_probe_forced_on_via_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(probe_mod.PROBE_ENV, "1") - probe = resolve_sandbox_probe() - assert isinstance(probe, FakeSandboxProbe) - assert probe.available() is True - - -def test_resolve_probe_defaults_to_real_probe(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(probe_mod.PROBE_ENV, raising=False) - assert isinstance(resolve_sandbox_probe(), SandboxProbe) - - -def test_real_probe_is_available_when_docker_info_succeeds() -> None: - def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - return _completed(0) - - assert SandboxProbe(runner=runner).available() is True - - -def test_real_probe_is_unavailable_when_docker_info_fails() -> None: - def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - return _completed(1) - - assert SandboxProbe(runner=runner).available() is False - - -@pytest.mark.parametrize( - "error", - [ - FileNotFoundError("docker"), - PermissionError("denied"), - OSError("cannot talk to daemon"), - subprocess.TimeoutExpired(cmd=["docker", "info"], timeout=5), - RuntimeError("unexpected probe failure"), - ], -) -def test_real_probe_returns_unavailable_without_raising(error: Exception) -> None: - def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - raise error - - assert SandboxProbe(runner=runner).available() is False - - -def test_real_probe_runs_docker_info_with_short_timeout() -> None: - seen: dict[str, Any] = {} - - def runner(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - seen["command"] = list(command) - seen["timeout"] = kwargs.get("timeout") - seen["check"] = kwargs.get("check") - return _completed(0) - - assert SandboxProbe(runner=runner).available() is True - assert seen["command"] == ["docker", "info"] - assert seen["timeout"] == probe_mod.PROBE_TIMEOUT_SECONDS - assert probe_mod.PROBE_TIMEOUT_SECONDS <= 10 - assert seen["check"] is False - - -def test_real_probe_is_skippable_in_unit_tests(monkeypatch: pytest.MonkeyPatch) -> None: - def boom(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - raise AssertionError("unit tests must not invoke the docker daemon") - - monkeypatch.setattr(probe_mod.subprocess, "run", boom) - - def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - return _completed(0) - - assert SandboxProbe(runner=runner).available() is True - - -def test_real_probe_default_path_uses_subprocess_run(monkeypatch: pytest.MonkeyPatch) -> None: - seen: dict[str, Any] = {} - - def fake_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - seen["command"] = list(command) - seen["timeout"] = kwargs.get("timeout") - seen["capture_output"] = kwargs.get("capture_output") - seen["text"] = kwargs.get("text") - return _completed(0) - - monkeypatch.setattr(probe_mod.subprocess, "run", fake_run) - assert SandboxProbe().available() is True - assert seen["command"] == ["docker", "info"] - assert seen["timeout"] == probe_mod.PROBE_TIMEOUT_SECONDS - assert seen["capture_output"] is True - assert seen["text"] is True diff --git a/tests/test_sandbox_runner.py b/tests/test_sandbox_runner.py deleted file mode 100644 index 0469847..0000000 --- a/tests/test_sandbox_runner.py +++ /dev/null @@ -1,364 +0,0 @@ -from __future__ import annotations - -import os -import stat -import subprocess -from pathlib import Path -from typing import Any - -import pytest - -from openultrasast.sandbox import ( - DockerCliRunner, - FakeSandboxRunner, - SandboxJob, - SandboxResult, - SandboxRunner, - build_docker_argv, - resolve_sandbox_runner, -) -from openultrasast.sandbox import runner as runner_mod - - -def _job(tmp_path: Path, **overrides: object) -> SandboxJob: - repo = tmp_path / "repo" - repo.mkdir(exist_ok=True) - (repo / "app.py").write_text("print(1)\n") - values: dict[str, object] = { - "image": "python:3.12-alpine", - "command": ("python", "/scratch/case.py"), - "repo_root": repo, - "scratch_files": {"case.py": "print('ok')\n"}, - "timeout_seconds": 30, - "memory_mb": 256, - "pids_limit": 128, - } - values.update(overrides) - return SandboxJob(**values) # type: ignore[arg-type] - - -def _flag_value(argv: list[str], flag: str) -> str: - try: - return argv[argv.index(flag) + 1] - except (ValueError, IndexError) as exc: - raise AssertionError(f"{flag} missing from {argv}") from exc - - -def _mount_options(spec: str) -> dict[str, str]: - options: dict[str, str] = {} - for part in spec.split(","): - if "=" in part: - key, value = part.split("=", 1) - options[key] = value - else: - options[part] = "true" - return options - - -def _workspace_mount(argv: list[str]) -> dict[str, str]: - specs: list[str] = [] - for index, item in enumerate(argv): - if item in {"--mount", "-v", "--volume"} and index + 1 < len(argv): - specs.append(argv[index + 1]) - for spec in specs: - options = _mount_options(spec) - target = options.get("dst") or options.get("destination") or options.get("target") - if target == "/workspace" or spec.endswith(":/workspace") or ":/workspace:" in spec: - return options if target == "/workspace" else {"raw": spec} - if "/workspace" in spec: - return options if options else {"raw": spec} - raise AssertionError(f"no /workspace mount in {argv}") - - -def _seed_source(argv: list[str]) -> Path: - mounts = [argv[index + 1] for index, item in enumerate(argv) if item == "--mount" and index + 1 < len(argv)] - seed_mounts = [spec for spec in mounts if "/scratch-in" in spec] - assert seed_mounts, argv - options = _mount_options(seed_mounts[0]) - source = Path(options.get("src") or options.get("source") or "") - assert source.is_dir(), argv - return source - - -def _copy_script(argv: list[str]) -> str: - try: - return argv[argv.index("-c") + 1] - except (ValueError, IndexError) as exc: - raise AssertionError(f"no /bin/sh -c wrapper in {argv}") from exc - - -def _posix_mode(path: Path) -> int: - return path.stat().st_mode & 0o777 - - -def test_build_docker_argv_starts_with_docker_run_rm_init(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - assert argv[0] == "docker" - assert argv[1] == "run" - assert "--rm" in argv - assert "--init" in argv - - -def test_build_docker_argv_includes_network_none_and_read_only(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - joined = " ".join(argv) - assert "--read-only" in argv - assert "--network none" in joined or "--network=none" in argv - assert "--network host" not in joined - assert "--network=host" not in argv - - -def test_build_docker_argv_excludes_docker_sock(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - joined = " ".join(argv) - assert "docker.sock" not in joined - assert "/var/run/docker.sock" not in joined - assert "-v" not in argv - assert "--volume" not in argv - - -def test_build_docker_argv_forbids_privileged_and_host_network(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - assert "--privileged" not in argv - assert "--pid" not in argv or _flag_value(argv, "--pid") != "host" - assert "--ipc" not in argv or _flag_value(argv, "--ipc") != "host" - assert "--userns" not in argv or "host" not in _flag_value(argv, "--userns") - - -def test_build_docker_argv_scratch_is_writable_tmpfs(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - tmpfs = _flag_value(argv, "--tmpfs") - assert tmpfs.startswith("/scratch") - assert "rw" in tmpfs - assert "exec" in tmpfs - - -def test_build_docker_argv_applies_memory_and_pids_limits(tmp_path: Path) -> None: - job = _job(tmp_path, memory_mb=512, pids_limit=64) - argv = build_docker_argv(job) - memory = _flag_value(argv, "--memory") - assert "512" in memory - assert memory.endswith("m") or memory.endswith("M") or memory.endswith("b") - assert _flag_value(argv, "--pids-limit") == "64" - - -def test_build_docker_argv_drops_capabilities_and_new_privileges(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - assert _flag_value(argv, "--cap-drop") == "ALL" - security = _flag_value(argv, "--security-opt") - assert "no-new-privileges" in security - - -def test_build_docker_argv_uses_non_root_user(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - user = _flag_value(argv, "--user") - uid = user.split(":", 1)[0] - assert uid not in {"0", "root"} - assert int(uid) != 0 - - -def test_build_docker_argv_bind_mounts_repo_read_only_at_workspace(tmp_path: Path) -> None: - job = _job(tmp_path) - argv = build_docker_argv(job) - mount = _workspace_mount(argv) - source = mount.get("src") or mount.get("source") - assert source == str(job.repo_root.resolve()) - assert mount.get("dst") == "/workspace" or mount.get("destination") == "/workspace" or mount.get("target") == "/workspace" - assert mount.get("ro") == "true" or mount.get("readonly") == "true" - assert "rw" not in mount - - -def test_build_docker_argv_source_tree_is_not_writable(tmp_path: Path) -> None: - argv = build_docker_argv(_job(tmp_path)) - mount = _workspace_mount(argv) - assert mount.get("ro") == "true" or mount.get("readonly") == "true" - assert "--read-only" in argv - joined = " ".join(argv) - assert f"{_job(tmp_path).repo_root.resolve()}:/workspace:rw" not in joined - assert f"{_job(tmp_path).repo_root.resolve()}:/workspace " not in joined + " " - - -def test_build_docker_argv_ends_with_image_and_command(tmp_path: Path) -> None: - job = _job(tmp_path) - argv = build_docker_argv(job) - assert argv[-len(job.command) :] == list(job.command) - assert argv[-len(job.command) - 1] == job.image - - -def test_fake_sandbox_runner_records_jobs_and_returns_programmed_result(tmp_path: Path) -> None: - programmed = SandboxResult(exit_code=7, stdout="out", stderr="err", timed_out=False) - runner: SandboxRunner = FakeSandboxRunner(programmed) - job = _job(tmp_path) - result = runner.run(job) - assert result == programmed - assert isinstance(runner, FakeSandboxRunner) - assert runner.jobs == [job] - - -def test_resolve_runner_fake_via_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(runner_mod.RUNNER_ENV, "fake") - assert isinstance(resolve_sandbox_runner(), FakeSandboxRunner) - - -def test_resolve_runner_defaults_to_docker_cli(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(runner_mod.RUNNER_ENV, raising=False) - assert isinstance(resolve_sandbox_runner(), DockerCliRunner) - - -def test_fake_sandbox_runner_can_be_reprogrammed(tmp_path: Path) -> None: - runner = FakeSandboxRunner() - first = _job(tmp_path, command=("echo", "one")) - second = _job(tmp_path, command=("echo", "two")) - timeout_result = SandboxResult(exit_code=-1, stdout="", stderr="deadline", timed_out=True) - runner.result = timeout_result - assert runner.run(first).timed_out is True - runner.result = SandboxResult(exit_code=0, stdout="ok", stderr="", timed_out=False) - assert runner.run(second).stdout == "ok" - assert runner.jobs == [first, second] - - -def test_fake_sandbox_runner_does_not_need_docker_daemon(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def boom(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - raise AssertionError("unit tests must not invoke the docker daemon") - - monkeypatch.setattr(subprocess, "run", boom) - monkeypatch.setattr(runner_mod.subprocess, "run", boom) - programmed = SandboxResult(exit_code=0, stdout="fake", stderr="", timed_out=False) - runner = FakeSandboxRunner(programmed) - assert runner.run(_job(tmp_path)) == programmed - - -def test_docker_cli_runner_uses_constructed_argv_and_timeout(tmp_path: Path) -> None: - seen: dict[str, Any] = {} - job = _job(tmp_path, scratch_files={}, timeout_seconds=19) - - def fake_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - seen["command"] = list(command) - seen["timeout"] = kwargs.get("timeout") - seen["check"] = kwargs.get("check") - seen["capture_output"] = kwargs.get("capture_output") - seen["text"] = kwargs.get("text") - return subprocess.CompletedProcess(command, 3, stdout="hello", stderr="warn") - - result = DockerCliRunner(runner=fake_run).run(job) - assert seen["command"] == build_docker_argv(job) - assert seen["timeout"] == 19 - assert seen["check"] is False - assert seen["capture_output"] is True - assert seen["text"] is True - assert result == SandboxResult(exit_code=3, stdout="hello", stderr="warn", timed_out=False) - assert "--network" in seen["command"] - assert "--read-only" in seen["command"] - assert "docker.sock" not in " ".join(seen["command"]) - - -def test_docker_cli_runner_maps_timeout_expired_to_timed_out(tmp_path: Path) -> None: - job = _job(tmp_path, scratch_files={}, timeout_seconds=5) - - def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: - raise subprocess.TimeoutExpired(cmd=command, timeout=5, output="partial", stderr="slow") - - result = DockerCliRunner(runner=fake_run).run(job) - assert result.timed_out is True - assert result.stdout == "partial" - assert result.stderr == "slow" - - -def test_docker_cli_runner_does_not_invoke_daemon_when_runner_injected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def boom(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: - raise AssertionError("unit tests must not invoke the docker daemon") - - monkeypatch.setattr(subprocess, "run", boom) - monkeypatch.setattr(runner_mod.subprocess, "run", boom) - - def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - - result = DockerCliRunner(runner=fake_run).run(_job(tmp_path, scratch_files={})) - assert result.timed_out is False - assert result.exit_code == 0 - - -def test_docker_cli_runner_does_not_write_the_source_tree(tmp_path: Path) -> None: - job = _job(tmp_path) - original = (job.repo_root / "app.py").read_text() - before = {path.relative_to(job.repo_root): path.read_text() for path in job.repo_root.rglob("*") if path.is_file()} - - def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: - joined = " ".join(command) - assert "docker.sock" not in joined - assert "--read-only" in command - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - - DockerCliRunner(runner=fake_run).run(job) - assert (job.repo_root / "app.py").read_text() == original - after = {path.relative_to(job.repo_root): path.read_text() for path in job.repo_root.rglob("*") if path.is_file()} - assert after == before - assert "case.py" not in {path.name for path in job.repo_root.iterdir()} - - -def test_docker_cli_runner_seeds_scratch_files_outside_the_repo(tmp_path: Path) -> None: - job = _job(tmp_path, scratch_files={"nested/case.py": "print('seeded')\n"}) - seen: dict[str, Any] = {} - - def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: - seen["command"] = list(command) - mounts = [command[index + 1] for index, item in enumerate(command) if item == "--mount" and index + 1 < len(command)] - seed_mounts = [spec for spec in mounts if "/scratch-in" in spec] - assert seed_mounts, command - options = _mount_options(seed_mounts[0]) - source = Path(options.get("src") or options.get("source") or "") - assert source.is_dir() - assert job.repo_root.resolve() not in source.resolve().parents - assert source.resolve() != job.repo_root.resolve() - assert (source / "nested" / "case.py").read_text() == "print('seeded')\n" - assert options.get("ro") == "true" or options.get("readonly") == "true" - assert "--tmpfs" in command - assert _flag_value(command, "--tmpfs").startswith("/scratch") - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - - result = DockerCliRunner(runner=fake_run).run(job) - assert result.exit_code == 0 - argv = seen["command"] - assert "docker.sock" not in " ".join(argv) - assert "--read-only" in argv - assert job.image in argv - - -def test_docker_cli_runner_scratch_seed_is_world_readable_for_container_user(tmp_path: Path) -> None: - job = _job(tmp_path, scratch_files={"nested/dir/case.py": "print('seeded')\n"}) - seen: dict[str, Path] = {} - - def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: - source = _seed_source(command) - seen["source"] = source - assert _posix_mode(source) == 0o755 - for dirpath, _dirnames, filenames in os.walk(source): - directory = Path(dirpath) - assert stat.S_ISDIR(directory.stat().st_mode) - assert _posix_mode(directory) == 0o755, directory - for name in filenames: - file_path = directory / name - assert _posix_mode(file_path) == 0o644, file_path - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - - DockerCliRunner(runner=fake_run).run(job) - assert "source" in seen - - -def test_docker_cli_runner_scratch_copy_uses_cp_r_not_cp_a(tmp_path: Path) -> None: - job = _job(tmp_path, scratch_files={"nested/case.py": "print('seeded')\n"}) - seen: dict[str, str] = {} - - def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: - script = _copy_script(command) - seen["script"] = script - assert command[command.index(job.image) + 1] == "/bin/sh" - assert "cp -a" not in script - assert "cp -a" not in " ".join(command) - assert script.startswith("cp -r /scratch-in/. /scratch/") - assert 'exec "$@"' in script - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - - DockerCliRunner(runner=fake_run).run(job) - assert seen["script"] == 'cp -r /scratch-in/. /scratch/ && exec "$@"' diff --git a/tests/test_scan_deep.py b/tests/test_scan_deep.py deleted file mode 100644 index 1a8af73..0000000 --- a/tests/test_scan_deep.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Deep mode: skip regression when the sandbox is missing; record verdicts with a fake runner (task 6.3).""" - -from __future__ import annotations - -import json -import subprocess -from pathlib import Path - -import pytest - -from openultrasast.cli import main -from openultrasast.sandbox import FakeSandboxRunner, SandboxResult -from openultrasast.sandbox import runner as sandbox_runner - -_VULN = "@app.route('/admin')\ndef admin():\n return eval(request.data)\n" - - -def _repo(tmp_path: Path) -> Path: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(_VULN) - return repo - - -def _fail(message: str): # type: ignore[no-untyped-def] - def _inner(*args: object, **kwargs: object) -> object: - raise AssertionError(message) - - return _inner - - -def _guard_docker(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: - docker_argv: list[list[str]] = [] - original_run = subprocess.run - - def guarded_run(command: object, *args: object, **kwargs: object) -> object: - argv = list(command) if isinstance(command, list | tuple) else [command] - if argv and Path(str(argv[0])).name == "docker": - docker_argv.append([str(item) for item in argv]) - raise AssertionError(f"deep scan test must not start docker: {argv}") - return original_run(command, *args, **kwargs) - - monkeypatch.setattr(subprocess, "run", guarded_run) - monkeypatch.setattr(sandbox_runner.DockerCliRunner, "run", _fail("deep scan test must not use DockerCliRunner")) - return docker_argv - - -def _latest_run(repo: Path) -> Path: - return sorted((repo / ".runs").iterdir())[-1] - - -def _crash_runner() -> FakeSandboxRunner: - return FakeSandboxRunner(SandboxResult(exit_code=139, stdout="", stderr="Segmentation fault\n", timed_out=False)) - - -def _enable_crash_sandbox(monkeypatch: pytest.MonkeyPatch) -> FakeSandboxRunner: - from openultrasast import cli - - fake = _crash_runner() - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake) - return fake - - -def test_deep_scan_skips_regress_when_sandbox_unavailable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - docker_argv = _guard_docker(monkeypatch) - - assert main(["scan", str(repo), "--mode", "deep"]) == 0 - - run_dir = _latest_run(repo) - manifest = json.loads((run_dir / "manifest.json").read_text()) - stages = manifest["stages"] - degradations = manifest.get("degradations", []) - - assert (run_dir / "complexity_map.json").is_file() - assert not (run_dir / "verdicts.json").exists() - assert docker_argv == [] - assert stages["requested"] == ["static", "map", "regress"] - assert stages["completed"] == ["static", "map"] - assert {"stage": "regress", "reason": "sandbox_unavailable"} in stages["skipped"] - assert any(entry.get("stage") == "regress" and entry.get("reason") == "sandbox_unavailable" for entry in degradations) - - -def test_deep_scan_with_fake_runner_writes_verdicts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_RUNNER", "fake") - docker_argv = _guard_docker(monkeypatch) - - assert main(["scan", str(repo), "--mode", "deep"]) == 0 - - run_dir = _latest_run(repo) - verdicts_path = run_dir / "verdicts.json" - manifest = json.loads((run_dir / "manifest.json").read_text()) - payload = json.loads(verdicts_path.read_text()) - stages = manifest["stages"] - - assert docker_argv == [] - assert verdicts_path.is_file() - assert (run_dir / "complexity_map.json").is_file() - assert stages["requested"] == ["static", "map", "regress"] - assert stages["completed"] == ["static", "map", "regress"] - assert not any(entry.get("reason") == "sandbox_unavailable" for entry in stages["skipped"]) - assert isinstance(payload["verdicts"], list) - assert payload["verdicts"] - for item in payload["verdicts"]: - assert item["verdict"] in {"triggerable", "not_triggerable", "already_covered", "inconclusive"} - assert "reason" in item - assert "path" in item - assert manifest["artifacts"]["verdicts"] == "verdicts.json" - assert manifest["artifacts"]["complexity_map"] == "complexity_map.json" - assert manifest["worth_fixing"]["count"] == sum(1 for item in payload["verdicts"] if item.get("worth_fixing")) - report = (run_dir / "report.md").read_text() - inventory_at = report.index("## Inventory") - map_at = report.index("## Complexity map") - worth_at = report.index("## Worth fixing") - assert inventory_at < map_at < worth_at - map_section = report[map_at:worth_at] - assert "app.py" in map_section - assert "Gap:" in map_section - ledger_path = repo / ".openultrasast" / "calibration" / "complexity_ledger.json" - assert ledger_path.is_file() - ledger = json.loads(ledger_path.read_text()) - assert ledger - assert any(entry.get("last_verdict") in {"triggerable", "not_triggerable"} for entry in ledger.values()) - - -def test_deep_scan_unsafe_hunter_snippet_is_safety_rejected_without_docker_job(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast import cli - from openultrasast.sandbox import FakeSandboxRunner - - repo = _repo(tmp_path) - config = tmp_path / "openultrasast.toml" - config.write_text('[models]\nhunter = "test-hunter"\n') - fake = FakeSandboxRunner() - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "unsafe-snippet") - monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake) - docker_argv = _guard_docker(monkeypatch) - - assert main(["scan", str(repo), "--mode", "deep", "--config", str(config)]) == 0 - - run_dir = _latest_run(repo) - payload = json.loads((run_dir / "verdicts.json").read_text()) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - hunter_findings = [item for item in findings if str(item["finding_id"]).startswith("tool-hunter:")] - - assert docker_argv == [] - assert hunter_findings - assert all(item["evidence_level"] == "suspicion" for item in hunter_findings) - hunter_ids = {item["finding_id"] for item in hunter_findings} - # Overlay promotions may still be proven; hunter suspicion never becomes a sandbox candidate. - for item in payload["verdicts"]: - assert hunter_ids.isdisjoint(item["inventory_finding_ids"]) - - -def test_deep_scan_fail_on_worth_fixing_exits_one_when_triggerable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path) - _enable_crash_sandbox(monkeypatch) - docker_argv = _guard_docker(monkeypatch) - - assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "worth-fixing"]) == 1 - assert docker_argv == [] - - run_dir = _latest_run(repo) - payload = json.loads((run_dir / "verdicts.json").read_text()) - manifest = json.loads((run_dir / "manifest.json").read_text()) - report = (run_dir / "report.md").read_text() - worth_fixing = [item for item in payload["verdicts"] if item.get("worth_fixing")] - - assert worth_fixing - assert len(worth_fixing) >= 1 - assert any(item["verdict"] == "triggerable" for item in worth_fixing) - assert manifest["worth_fixing"]["count"] == len(worth_fixing) - assert "## Complexity map" in report - assert "## Worth fixing" in report - assert "triggerable" in report - ledger = json.loads((repo / ".openultrasast" / "calibration" / "complexity_ledger.json").read_text()) - assert any(entry.get("last_verdict") == "triggerable" for entry in ledger.values()) - - -def test_deep_scan_fail_on_never_exits_zero_with_worth_fixing_verdict(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path) - _enable_crash_sandbox(monkeypatch) - docker_argv = _guard_docker(monkeypatch) - - assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "never"]) == 0 - assert docker_argv == [] - - payload = json.loads((_latest_run(repo) / "verdicts.json").read_text()) - assert any(item.get("worth_fixing") for item in payload["verdicts"]) diff --git a/tests/test_search_demo_replay.py b/tests/test_search_demo_replay.py index ae5f7cd..5e87572 100644 --- a/tests/test_search_demo_replay.py +++ b/tests/test_search_demo_replay.py @@ -196,13 +196,6 @@ def test_lane_failure_is_could_not_run(tmp_path, monkeypatch): assert "verifier unavailable" in record["reason"] -def test_replay_registered_as_standalone(): - from openultrasast.model.audit import audit - - rows = {r.module: r for r in audit(Path("benchmarks"))} - assert rows["search.demo_replay"].classification == "standalone_capability" - - def test_docker_selection_uses_same_preparation_and_fresh_runs(tmp_path, monkeypatch): args, root, fake_lane, events = harness(tmp_path, monkeypatch) args.lane = "docker" diff --git a/tests/test_semantic_pipeline.py b/tests/test_semantic_pipeline.py index 2b2fdd8..f85bd5b 100644 --- a/tests/test_semantic_pipeline.py +++ b/tests/test_semantic_pipeline.py @@ -125,46 +125,6 @@ def test_prove_filter_drops_demoted_keeps_promoted() -> None: assert [item.finding_id for item in promoted_findings(findings, records)] == [promoted_id] -def test_deep_scan_offers_only_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast import cli - from openultrasast.sandbox import FakeSandboxRunner, SandboxResult - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def safe():\n exec('ls')\n\ndef bad():\n return eval(request.args)\n") - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False)) - monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake) - assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "worth-fixing"]) == 0 - run_dir = _latest_run(repo) - overlay = json.loads((run_dir / "overlay.json").read_text())["records"] - dispositions = {item["proposal_id"]: item["disposition"] for item in overlay} - demoted = [key for key, value in dispositions.items() if value == "demote"] - promoted = [key for key, value in dispositions.items() if value == "promote"] - assert demoted - assert promoted - verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] - offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]} - assert offered <= set(promoted) - assert not (offered & set(demoted)) - - -def test_fail_on_worth_fixing_ignores_unadjudicated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "broken.py").write_text("eval(request.args)\ndef oops(\n") - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard", "--fail-on", "worth-fixing"]) == 0 - run_dir = _latest_run(repo) - overlay = json.loads((run_dir / "overlay.json").read_text())["records"] - assert overlay - assert all(item["disposition"] == "unadjudicated" for item in overlay) - report = (run_dir / "report.md").read_text() - assert "unadjudicated" in report - - def test_unlabeled_messy_tree_has_both_dispositions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo = tmp_path / "repo" repo.mkdir() diff --git a/tests/test_skills.py b/tests/test_skills.py deleted file mode 100644 index 28ead94..0000000 --- a/tests/test_skills.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Trail of Bits skill index + router (Phase 14).""" - -from openultrasast.preprocess import FileTarget -from openultrasast.skills import ( - SKILL_INVENTORY, - route_skill_ids, - route_skills, - route_skills_for_target, - select_skill_context, - skill_chunks, -) - - -def _target(language: str, tags: list[str]) -> FileTarget: - return FileTarget(path="t", absolute_path="/repo/t", language=language, loc=10, tags=tags, has_fuzz_entry_point=False) - - -# ---- routing by language / tags / vulnerability class ----------------------- - - -def test_c_parser_target_routes_to_memory_and_fuzzing_skills() -> None: - ids = route_skill_ids(_target("c", ["memory_unsafe", "parser"])) - assert {"address-sanitizer", "harness-writing"} <= set(ids) # memory + fuzzing - assert "libfuzzer" in ids - - -def test_crypto_target_routes_to_constant_time_and_vector_skills() -> None: - ids = route_skill_ids(_target("python", ["crypto"])) - assert "constant-time-analysis" in ids - assert "vector-forge" in ids - - -def test_language_mismatch_excludes_other_language_skills() -> None: - # A C target must not pull Python-only (atheris) or Rust-only (cargo-fuzz) skills. - ids = route_skill_ids(_target("c", ["memory_unsafe", "parser", "fuzzable"])) - assert "atheris" not in ids - assert "cargo-fuzz" not in ids - - -def test_language_only_match_does_not_select() -> None: - # python + crypto must not surface atheris (python but no crypto signal). - ids = route_skills_for_target(_target("python", ["crypto"]), max_snippets=10) - assert "atheris" not in [skill.id for skill in ids] - - -# ---- routing by stage ------------------------------------------------------- - - -def test_stage_filter_restricts_to_stage_skills() -> None: - # fuzzing-dictionary is verify-only; at the map stage it must not appear, while a - # map-stage skill (semgrep) does for an injection signal. - verify = route_skills(tags=("parser",), vulnerability_classes=("parsing",), stage="verify", max_snippets=10) - mapped = route_skills(tags=("parser",), vulnerability_classes=("parsing",), stage="map", max_snippets=10) - assert "fuzzing-dictionary" in {s.id for s in verify} - assert "fuzzing-dictionary" not in {s.id for s in mapped} - - -# ---- prompt budget caps ----------------------------------------------------- - - -def test_select_skill_context_respects_budget() -> None: - target = _target("c", ["memory_unsafe", "parser"]) - full = select_skill_context(target, budget_chars=10_000) - capped = select_skill_context(target, budget_chars=80) - assert len(capped) <= 80 - assert len(full) > len(capped) - - -def test_zero_budget_yields_no_context() -> None: - assert select_skill_context(_target("c", ["memory_unsafe"]), budget_chars=0) == "" - - -# ---- retrieval-namespace chunking + inventory integrity --------------------- - - -def test_skill_chunks_land_in_the_skills_namespace() -> None: - chunks = skill_chunks() - assert chunks - assert all(chunk.namespace == "skills" for chunk in chunks) - assert any(chunk.metadata.get("skill_id") == "address-sanitizer" for chunk in chunks) - - -def test_inventory_ids_are_unique() -> None: - ids = [skill.id for skill in SKILL_INVENTORY] - assert len(ids) == len(set(ids)) diff --git a/tests/test_split_sink_regress.py b/tests/test_split_sink_regress.py deleted file mode 100644 index 8eac7a9..0000000 --- a/tests/test_split_sink_regress.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Stage-3 regression on split-sink: fake runner always; live Docker skips without a daemon.""" - -from pathlib import Path - -import pytest - -from openultrasast.config import SandboxConfig -from openultrasast.regress.recipes import recipe_for -from openultrasast.regress.safety import check_snippet_safety -from openultrasast.regress.verdict import TRIGGERABLE, verdict_from_result -from openultrasast.sandbox import DockerCliRunner, FakeSandboxRunner, SandboxProbe, SandboxResult - -_SPLIT_SINK_PYTHON = Path("benchmarks/fixtures/split-sink-python") -_LIMITS = SandboxConfig(memory_mb=256, timeout_seconds=30, pids_limit=128) -_PYTHON_IMAGE = "python:3.12-alpine" -_SNIPPET = 'term = "x\'; drop"\nquery = "select * from items where title like \'%" + term + "%\'"\nassert "\'" not in query\n' - - -def _split_sink_job(): - check_snippet_safety(_SNIPPET) - job = recipe_for("python", _SNIPPET, _PYTHON_IMAGE, _LIMITS, repo_root=_SPLIT_SINK_PYTHON.resolve()) - assert job is not None - return job - - -def test_fake_runner_marks_split_sink_snippet_triggerable() -> None: - fake = FakeSandboxRunner(SandboxResult(exit_code=1, stdout="", stderr="AssertionError", timed_out=False)) - result = fake.run(_split_sink_job()) - verdict = verdict_from_result(result) - assert verdict.verdict == TRIGGERABLE - assert fake.jobs - - -@pytest.mark.docker -def test_docker_marks_split_sink_python_triggerable() -> None: - if not SandboxProbe().available(): - pytest.skip("docker daemon unavailable") - result = DockerCliRunner().run(_split_sink_job()) - verdict = verdict_from_result(result) - assert verdict.verdict == TRIGGERABLE, (result.exit_code, result.stdout, result.stderr) diff --git a/tests/test_stages.py b/tests/test_stages.py index 207e9b8..12fd829 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -36,15 +36,15 @@ def test_plan_for_mode_requests_ordered_stages() -> None: def test_skipped_stage_is_structured_degradation_not_exception() -> None: plan = plan_for_mode("deep") - skipped = record_skip(plan, Stage.REGRESS, "sandbox_unavailable") + skipped = record_skip(plan, Stage.REGRESS, "regress_retired") assert skipped.requested == (Stage.STATIC, Stage.MAP, Stage.REGRESS) assert skipped.completed == () - assert skipped.skipped == ({"stage": "regress", "reason": "sandbox_unavailable"},) + assert skipped.skipped == ({"stage": "regress", "reason": "regress_retired"},) degradations = list(skipped.skipped) assert degradations[0]["stage"] == "regress" - assert degradations[0]["reason"] == "sandbox_unavailable" - assert json.dumps(degradations) == json.dumps([{"stage": "regress", "reason": "sandbox_unavailable"}]) + assert degradations[0]["reason"] == "regress_retired" + assert json.dumps(degradations) == json.dumps([{"stage": "regress", "reason": "regress_retired"}]) def test_record_completed_static_for_quick() -> None: diff --git a/tests/test_tool_hunter.py b/tests/test_tool_hunter.py index 8269cb6..e123b7b 100644 --- a/tests/test_tool_hunter.py +++ b/tests/test_tool_hunter.py @@ -198,8 +198,6 @@ def _latest_run(repo: Path) -> Path: def _guard_docker(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: import subprocess - from openultrasast.sandbox import runner as sandbox_runner - docker_argv: list[list[str]] = [] original_run = subprocess.run @@ -210,11 +208,7 @@ def _guard_docker(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: raise AssertionError(f"hunter scan test must not start docker: {argv}") return original_run(command, *args, **kwargs) - def fail(*args: object, **kwargs: object) -> object: - raise AssertionError("hunter scan test must not use DockerCliRunner") - monkeypatch.setattr(subprocess, "run", guarded_run) - monkeypatch.setattr(sandbox_runner.DockerCliRunner, "run", fail) return docker_argv