mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Fable 5.1
f63703a604
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1486 lines
69 KiB
Python
1486 lines
69 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections.abc import Mapping, Sequence
|
|
from dataclasses import asdict, dataclass, replace
|
|
from pathlib import Path
|
|
from time import perf_counter
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from . import tool_hunter
|
|
from .benchmark import (
|
|
create_benchmark_run,
|
|
evaluate_benchmark,
|
|
load_baseline_findings,
|
|
load_benchmark_manifest,
|
|
load_findings,
|
|
resolve_benchmark_source,
|
|
write_benchmark_artifacts,
|
|
)
|
|
from .calibration import (
|
|
FalsePositiveLearning,
|
|
calibrate_rankings,
|
|
fp_reachability_overrides,
|
|
learnings_from_verifications,
|
|
load_false_positive_learnings,
|
|
merge_false_positive_learnings,
|
|
write_false_positive_learnings,
|
|
write_ranking_calibrations,
|
|
)
|
|
from .complexity import map as complexity_map
|
|
from .config import ModelLayerConfig, ObligationsConfig, ResolvedConfig, RetiredConfigError, load_config, load_dotenv
|
|
from .findings import StaticFinding, quick_scan_findings, uncovered_languages, write_findings
|
|
from .fusion import FusionDecision, fuse_findings
|
|
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
|
from .hunter import run_hunter_pool, write_hunter_trajectories
|
|
from .learn.cli import add_commands as add_learn_commands
|
|
from .learn.cli import run as run_learn
|
|
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
|
from .pair_gate import print_pair_metrics
|
|
from .pairs import (
|
|
DEFAULT_CATALOG,
|
|
SLICE_NAMES,
|
|
evaluate_catalog,
|
|
load_pair_catalog,
|
|
result_payload,
|
|
select_profile,
|
|
select_slice,
|
|
select_split,
|
|
select_vendored,
|
|
)
|
|
from .policy import assert_rules_resolve, load_policy
|
|
from .preprocess import FileTarget, preprocess_repository, write_preprocess_artifact
|
|
from .profiling import start as start_profiling
|
|
from .provenance import fingerprint
|
|
from .rank import rank_obligations, rank_targets, write_rankings
|
|
from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report
|
|
from .repos import (
|
|
DEFAULT_REPO_DIR,
|
|
RepoUnavailable,
|
|
checkout_path,
|
|
load_repo_recipes,
|
|
recipe_payload,
|
|
)
|
|
from .repos import resolve as resolve_repo
|
|
from .repos import select as select_repo
|
|
from .repos import verify as verify_repo
|
|
from .ruleset import DEFAULT_RULESET_DIR, load_ruleset
|
|
from .run import ScanRun, create_scan_run
|
|
from .scoring import build_score_artifact
|
|
from .semantic import (
|
|
OverlayRecord,
|
|
adjudicate,
|
|
finding_from_coverage,
|
|
load_facts,
|
|
write_overlay,
|
|
)
|
|
from .semantic.facts import FactLoadError, SemanticFacts
|
|
from .stages import Stage, plan_for_mode, record_completed, record_skip, skip_as_degradation, stages_payload
|
|
from .verification import VerificationResult, verify_findings, write_verification_results
|
|
|
|
CALIBRATION_DIR = ".openultrasast/calibration"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ScanOutcome:
|
|
scan_id: str
|
|
run_dir: Path
|
|
file_target_count: int
|
|
ranked_target_count: int
|
|
finding_count: int
|
|
calibrations_applied: int
|
|
exit_code: int
|
|
notices: tuple[str, ...] = ()
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
pass
|
|
|
|
|
|
def _configure_logging() -> None:
|
|
"""Send logs to stderr at the level the environment asks for; quiet by default."""
|
|
level = os.environ.get("OUSAST_LOG_LEVEL", "").strip().upper()
|
|
if level not in {"DEBUG", "INFO", "WARNING", "ERROR"}:
|
|
return
|
|
handler = logging.StreamHandler()
|
|
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s", "%H:%M:%S"))
|
|
root = logging.getLogger("openultrasast")
|
|
root.handlers = [handler]
|
|
root.setLevel(getattr(logging, level))
|
|
root.propagate = False
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
try:
|
|
return _main(argv)
|
|
except RetiredConfigError as exc: # a retired key is an error that names its replacement, exit 2
|
|
print(f"ousast: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
def _main(argv: list[str] | None) -> int:
|
|
given = list(argv if argv is not None else sys.argv[1:])
|
|
if given[:1] == ["pre-push"] and given[1:2] in (["install"], ["uninstall"]):
|
|
return _hook_install(given[1], given[2:])
|
|
if given[:1] != ["pre-push"]:
|
|
load_dotenv()
|
|
parser = argparse.ArgumentParser(prog="ousast")
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
replay_parser = subparsers.add_parser(
|
|
"pre-push",
|
|
help="check Git push input or replay local revisions (experimental, advisory)",
|
|
description="Check what a push changes: quick rules on the changed lines, then the Joern engine on head and base "
|
|
"within one deadline. Advisory: exit 0 unless --mode blocking. Install the Git hook with "
|
|
"'ousast pre-push install [REPO] [--force] [--docker]'; remove it with 'ousast pre-push uninstall [REPO]'.",
|
|
)
|
|
replay_parser.add_argument("path", type=Path, help="the repository to check (the hook passes '.')")
|
|
replay_parser.add_argument("--base", help="explicit replay: the base revision (with --head; instead of --remote)")
|
|
replay_parser.add_argument("--head", help="explicit replay: the head revision to check against --base")
|
|
replay_parser.add_argument(
|
|
"--remote",
|
|
nargs=2,
|
|
metavar=("NAME", "URL"),
|
|
help="Git hook mode: the remote name and URL Git passes to pre-push; the push input is read from stdin",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--comparison-base",
|
|
help="hook mode: a local ref to compare a NEW branch against (the remote has no base for it); "
|
|
"the hook reads OUSAST_COMPARISON_BASE",
|
|
)
|
|
replay_parser.add_argument("--model-config", type=Path, help="explicit optional witness selection endpoint configuration")
|
|
replay_parser.add_argument("--prior-hook", type=Path, help="explicitly chain an existing hook with the same input and arguments")
|
|
replay_parser.add_argument(
|
|
"--artifact", type=Path, required=True, help="where to write the complete JSON result; must be outside the repository"
|
|
)
|
|
replay_parser.add_argument(
|
|
"--cache-dir", type=Path, help="reuse compatible local artifacts in a private directory outside the repository"
|
|
)
|
|
replay_parser.add_argument(
|
|
"--deadline",
|
|
type=float,
|
|
default=30.0,
|
|
help="seconds for the whole check: resolution, quick rules, engine and reporting (default 30; hook: OUSAST_PUSH_DEADLINE)",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--cancellation-allowance",
|
|
type=float,
|
|
default=2.0,
|
|
help="seconds after the deadline to stop work and save the result (default 2)",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--max-regions", type=int, default=500, help="most code regions the engine examines, highest-ranked first (default 500)"
|
|
)
|
|
replay_parser.add_argument(
|
|
"--mode",
|
|
choices=("advisory", "blocking"),
|
|
default="advisory",
|
|
help="advisory (default) never blocks; blocking is the explicit opt-in that rejects a push for an admitted "
|
|
"alert (hook: OUSAST_PUSH_MODE)",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--incomplete-coverage",
|
|
choices=("allow", "block"),
|
|
default="allow",
|
|
help="with --mode blocking only: also block when coverage is incomplete (hook: OUSAST_INCOMPLETE_COVERAGE)",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--engine",
|
|
choices=("inline", "background", "off"),
|
|
default="inline",
|
|
help="inline (default): run the Joern engine inside the deadline; background: return after the quick rules and "
|
|
"run the engine detached, showing its result on the next run; off: quick rules only",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--background-deadline",
|
|
type=float,
|
|
default=900.0,
|
|
help="seconds the detached engine run of --engine background may take (default 900)",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--experimental-declarations",
|
|
type=Path,
|
|
help="explicit --base/--head replay only: unreviewed capability declarations used to render the experimental "
|
|
"evaluation explanation; never consulted by admission and cannot enable any capability",
|
|
)
|
|
replay_parser.add_argument(
|
|
"--experimental-record-vetoes",
|
|
action="store_true",
|
|
help="explicit --base/--head replay only: record each comparison and admission veto per raw finding instead of "
|
|
"applying it; output is labeled experimental, changes no result and enables no hook capability",
|
|
)
|
|
|
|
scan = subparsers.add_parser("scan", help="scan a local repository")
|
|
scan.add_argument("path", type=Path)
|
|
scan.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick")
|
|
scan.add_argument("--config", type=Path, default=Path("openultrasast.toml"))
|
|
scan.add_argument("--fail-on", choices=("never", "findings", "verified"), default="never")
|
|
|
|
benchmark = subparsers.add_parser("benchmark", help="run a benchmark manifest and write scoreboard artifacts")
|
|
benchmark.add_argument("manifest", type=Path)
|
|
benchmark.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick")
|
|
benchmark.add_argument("--config", type=Path, default=Path("openultrasast.toml"))
|
|
|
|
pairs = subparsers.add_parser(
|
|
"pairs",
|
|
help="scan isolated vuln-vs-fixed pairs (TP on vuln, silent on fix) and report misses and fixed-side false positives",
|
|
)
|
|
pairs.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
|
pairs.add_argument("--slice", choices=SLICE_NAMES, default="all")
|
|
pairs.add_argument(
|
|
"--profile", choices=("all", "human", "agent", "mixed", "synthetic"), default="all", help="filter pairs by provenance profile"
|
|
)
|
|
pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split")
|
|
pairs.add_argument("--pointers", action="store_true", help="allow network for non-vendored pointer pairs this run (nightly; CI never)")
|
|
pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON")
|
|
|
|
repos = subparsers.add_parser(
|
|
"repos",
|
|
help="pinned known-vulnerable checkouts: list them, and verify a recipe against the code it names",
|
|
)
|
|
repos.add_argument("--dir", type=Path, default=DEFAULT_REPO_DIR)
|
|
repos.add_argument("--repo", default="all", help="one recipe by name, or all")
|
|
repos.add_argument("--fetch", action="store_true", help="allow network to materialise missing checkouts (CI never)")
|
|
repos.add_argument("--json", action="store_true")
|
|
|
|
subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
|
|
model_cmd = subparsers.add_parser("model", help="maintainer: the model layer that arbitrates detector claims")
|
|
model_sub = model_cmd.add_subparsers(dest="model_command", required=True)
|
|
candidates = model_sub.add_parser("candidates", help="offline: what the candidate enumerator can reach, per family and per slice")
|
|
candidates.add_argument("--out", type=Path, default=Path(".openultrasast/model"), help="where artifacts are written")
|
|
candidates.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
|
candidates.add_argument("--slice", choices=SLICE_NAMES, default="all")
|
|
candidates.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites")
|
|
candidates.add_argument("--json", action="store_true")
|
|
|
|
learn = subparsers.add_parser("learn", help="maintainer: the learned decision engine's data (labels)")
|
|
learn_sub = learn.add_subparsers(dest="learn_command", required=True)
|
|
learn_labels = learn_sub.add_parser("labels", help="labels from the fail-closed sources in learn/sources.toml (host only)")
|
|
learn_labels.add_argument("--sources", type=Path, help="the source list (default: the packaged learn/sources.toml)")
|
|
learn_labels.add_argument("--root", type=Path, default=Path("."), help="the repository root the source paths are relative to")
|
|
learn_labels.add_argument("--cache", type=Path, help="clones and checkouts (default ~/.cache/openultrasast)")
|
|
learn_labels.add_argument("--out", type=Path, help="write the full rows to OUT/labels-<sha>.jsonl (local: they name functions)")
|
|
learn_labels.add_argument("--snapshot", type=Path, help="write the counts-only snapshot (the committable record) here")
|
|
learn_labels.add_argument("--include-title", action="store_true", help="the title-tier pairs as well (a sensitivity arm)")
|
|
learn_labels.add_argument("--no-assumed-benign", action="store_true", help="skip mining the assumed-benign history source")
|
|
add_learn_commands(learn_sub)
|
|
plane = subparsers.add_parser("plane", help="maintainer: the ax-backed service plane on the profile's cluster (ops/k8s/profiles/)")
|
|
profile_help = "the PlaneProfile: a name under ops/k8s/profiles/ or a .toml path (default OUSAST_PLANE_PROFILE, else kind)"
|
|
plane_sub = plane.add_subparsers(dest="plane_command", required=True)
|
|
plane_run = plane_sub.add_parser("run", help="execute a Run manifest on ax; a rerun skips done tasks")
|
|
plane_run.add_argument("run_manifest", type=Path)
|
|
plane_run.add_argument("--workers", type=int, default=1, help="tasks in flight at once (serialize labels never overlap)")
|
|
plane_run.add_argument("--ax", default="ax", help="path of the ax CLI")
|
|
plane_run.add_argument("--profile", help=profile_help)
|
|
plane_status = plane_sub.add_parser("status", help="per-task status and the token attribution table of a run")
|
|
plane_status.add_argument("run")
|
|
plane_status.add_argument("--units", action="store_true", help="per-unit rows from each task's units.jsonl")
|
|
plane_doctor = plane_sub.add_parser(
|
|
"doctor", help="check the profile's cluster, Agent Substrate, the ax controller, the runner image and the memory store"
|
|
)
|
|
plane_doctor.add_argument("--profile", help=profile_help)
|
|
plane_remember = plane_sub.add_parser("remember", help="ingest a run's memory rows into the store (OUSAST_MEMORY)")
|
|
plane_remember.add_argument("run")
|
|
plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/<run>.yaml names population and split")
|
|
plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)")
|
|
plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)")
|
|
plane_normalise = plane_sub.add_parser(
|
|
"memory-normalise", help='rewrite the store\'s rows to the fixed schema (every queryable field, "" where absent); idempotent'
|
|
)
|
|
plane_normalise.add_argument("--store", help="the store URL (default OUSAST_MEMORY): file:///<path> or s3://<bucket>[/<prefix>]")
|
|
plane_normalise.add_argument("--dry-run", action="store_true", help="count what would be rewritten, write nothing")
|
|
plane_engine = plane_sub.add_parser(
|
|
"alerts-engine", help="a Run's `alerts` for PHP and languages quick mode does not cover, from the engine image on this host"
|
|
)
|
|
plane_engine.add_argument("run_manifest", type=Path)
|
|
plane_engine.add_argument(
|
|
"--source", type=Path, required=True, help="a frozen export: git archive HEAD src benchmarks/push/finding_dump.py"
|
|
)
|
|
plane_engine.add_argument("--cases", default="", help="comma-separated case ids (default: every alerts case of the Run)")
|
|
plane_engine.add_argument("--deadline", type=float, default=1800.0, help="the engine's scan deadline per pin, seconds")
|
|
plane_engine.add_argument("--image", default="openultrasast:dev")
|
|
plane_engine.add_argument("--repos", type=Path, help="the case cache (default ~/.cache/openultrasast/independent)")
|
|
plane_engine.add_argument("--work", type=Path, help="where pins are exported and results land (default: a temporary directory)")
|
|
plane_harvest = plane_sub.add_parser(
|
|
"harvest", help="the decision engine's harvest Runs (verify a/b + agree, model roles) over a label snapshot"
|
|
)
|
|
plane_harvest.add_argument("--labels", type=Path, required=True, help="the full label rows (`ousast learn labels --out`)")
|
|
plane_harvest.add_argument(
|
|
"--plane", type=Path, required=True, help="where to write runs/, tasks/, workspaces/, models/, units.json (scratch)"
|
|
)
|
|
plane_harvest.add_argument("--name", default="harvest", help="Run name prefix: <name>-verify and <name>-roles")
|
|
plane_harvest.add_argument("--templates", type=Path, default=Path("plane"), help="the plane root holding tasks/ and models/ templates")
|
|
plane_harvest.add_argument("--catalog", type=Path, default=Path("benchmarks/pairs/catalog.toml"))
|
|
plane_harvest.add_argument(
|
|
"--cache", type=Path, default=Path.home() / ".cache" / "openultrasast", help="case clones and recipe checkouts"
|
|
)
|
|
plane_harvest.add_argument("--ceiling", type=float, default=10.0, help="USD: each Run's task budgets sum to at most this")
|
|
plane_harvest.add_argument("--runner-image", type=Path, help="re-pin the task templates to the image in this file first (ops/ax/up.sh)")
|
|
plane_harvest.add_argument("--profile", help=profile_help)
|
|
plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
|
|
plane_ws.add_argument("population", type=Path)
|
|
plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
|
|
plane_ws.add_argument("--profile", help=profile_help)
|
|
plane_ws.add_argument("--validation-set", type=Path, help="JSON {case: [[path, function], ...]}: emit that set's increment instead")
|
|
plane_ws.add_argument("--candidates", type=Path, help="with --validation-set: per-case candidate records (independent-v2-modelsinks)")
|
|
plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
|
|
plane_ws.add_argument("--run", default="validation-46", help="with --validation-set: the Run name")
|
|
plane_ws.add_argument("--plane", type=Path, default=Path("plane"), help="with --validation-set: the manifest root")
|
|
plane_ws.add_argument(
|
|
"--repos", type=Path, default=Path.home() / ".cache" / "openultrasast" / "independent",
|
|
help="with --validation-set: one clone per case id, for the fix ranges agree scores by (evaluate.hunks)",
|
|
) # fmt: skip
|
|
plane_ws.add_argument(
|
|
"--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)"
|
|
)
|
|
raw = list(argv if argv is not None else sys.argv[1:])
|
|
if _advisory_hook(raw):
|
|
# Git hook mode (stdin) without an explicit blocking opt-in: a check that cannot run must
|
|
# never reject the push. Say so in one line and allow it (Requirement 9.1).
|
|
try:
|
|
return _hook_main(parser, argv)
|
|
except SystemExit as stop:
|
|
if stop.code in (0, None):
|
|
raise
|
|
print("ousast: pre-push check did not run: invalid arguments or settings (message above). Push continues.")
|
|
return 0
|
|
except Exception as error: # noqa: BLE001 -- an advisory hook reports a crash, it never blocks on one
|
|
print(f"ousast: pre-push check did not run: unexpected {type(error).__name__}. Push continues.")
|
|
return 0
|
|
args = parser.parse_args(argv)
|
|
# Diagnosis needs the stage costs, and nothing configures logging, so the default root level of
|
|
# WARNING silently dropped every informational line. A run that reports nothing is
|
|
# indistinguishable from a run with nothing to report, which is how four PHP investigations were
|
|
# spent reading a later stage's timeout as the cause.
|
|
_configure_logging()
|
|
start_profiling() # only when OUSAST_SAMPLE_PROFILE names a file
|
|
if args.command == "pre-push":
|
|
return _pre_push(args, parser)
|
|
if args.command == "scan":
|
|
return _scan(args.path, args.config, args.mode, args.fail_on)
|
|
if args.command == "benchmark":
|
|
return _benchmark(args.manifest, args.config, args.mode)
|
|
if args.command == "pairs":
|
|
return _pairs(
|
|
args.catalog,
|
|
args.slice,
|
|
json_out=args.json,
|
|
profile=args.profile,
|
|
split=args.split,
|
|
pointers=args.pointers,
|
|
)
|
|
if args.command == "repos":
|
|
return _repos(args.dir, args.repo, fetch=args.fetch, json_out=args.json)
|
|
if args.command == "model":
|
|
return _model_candidates(args)
|
|
if args.command == "plane":
|
|
return _plane(args)
|
|
if args.command == "learn":
|
|
return _learn_labels(args) if args.learn_command == "labels" else run_learn(args)
|
|
if args.command == "mcp":
|
|
from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional
|
|
|
|
return serve()
|
|
return 2
|
|
|
|
|
|
def _hook_install(action: str, argv: list[str]) -> int:
|
|
"""`ousast pre-push install|uninstall [REPO]` from the hook shipped in the package (task 17.6)."""
|
|
from .push.install import install, uninstall
|
|
|
|
parser = argparse.ArgumentParser(
|
|
prog=f"ousast pre-push {action}",
|
|
description="Install the advisory OpenUltraSAST pre-push hook into a repository's effective hook directory "
|
|
"(core.hooksPath is honoured, never edited)."
|
|
if action == "install"
|
|
else "Remove the OpenUltraSAST pre-push hook and restore a chained previous hook.",
|
|
)
|
|
parser.add_argument("repository", nargs="?", type=Path, default=Path("."), help="the repository (default: current directory)")
|
|
parser.add_argument(
|
|
"--force",
|
|
action="store_true",
|
|
help="install: replace our hook, or move a foreign hook to pre-push.before-ousast and chain it; "
|
|
"uninstall: remove a hook this tool did not write",
|
|
)
|
|
if action == "install":
|
|
parser.add_argument(
|
|
"--docker",
|
|
action="store_true",
|
|
help="also install the ousast-docker wrapper beside the hook, so the check runs in the container image",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
code, lines = (
|
|
install(args.repository, force=args.force, docker=args.docker)
|
|
if action == "install"
|
|
else uninstall(args.repository, force=args.force)
|
|
)
|
|
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
print(f"ousast pre-push {action}: {error}", file=sys.stderr)
|
|
return 1
|
|
for line in lines:
|
|
print(line, file=sys.stderr if code else sys.stdout)
|
|
return code
|
|
|
|
|
|
def _advisory_hook(argv: list[str]) -> bool:
|
|
"""A Git hook invocation (`--remote NAME URL`, stdin input) that did not opt into blocking."""
|
|
if argv[:1] != ["pre-push"] or "--remote" not in argv:
|
|
return False
|
|
blocking = any(a == "--mode=blocking" for a in argv) or any(
|
|
a == "--mode" and i + 1 < len(argv) and argv[i + 1] == "blocking" for i, a in enumerate(argv)
|
|
)
|
|
return not blocking
|
|
|
|
|
|
def _hook_main(parser: argparse.ArgumentParser, argv: list[str] | None) -> int:
|
|
args = parser.parse_args(argv)
|
|
_configure_logging()
|
|
return _pre_push(args, parser)
|
|
|
|
|
|
def _pre_push(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
from .config import PushConfig
|
|
from .push.runner import push, replay
|
|
|
|
try:
|
|
if bool(args.base) != bool(args.head) or bool(args.base) == bool(args.remote):
|
|
parser.error("provide --base and --head together, or --remote NAME URL for Git stdin")
|
|
settings = PushConfig(
|
|
comparison_base=args.comparison_base,
|
|
deadline_seconds=args.deadline,
|
|
cancellation_allowance_seconds=args.cancellation_allowance,
|
|
mode=args.mode,
|
|
incomplete_coverage_policy=args.incomplete_coverage,
|
|
)
|
|
options: dict[str, Any] = dict(
|
|
artifact=args.artifact,
|
|
config=settings,
|
|
max_regions=args.max_regions,
|
|
cache_dir=args.cache_dir,
|
|
model_config=args.model_config,
|
|
)
|
|
if args.base and args.prior_hook:
|
|
parser.error("--prior-hook requires Git stdin mode")
|
|
if args.experimental_record_vetoes and not args.base:
|
|
parser.error("--experimental-record-vetoes requires explicit --base/--head replay; it is never a hook capability")
|
|
if args.experimental_declarations and not args.experimental_record_vetoes:
|
|
parser.error("--experimental-declarations requires --experimental-record-vetoes")
|
|
prior_data = None
|
|
if args.base:
|
|
delivery = replay(
|
|
args.path,
|
|
base=args.base,
|
|
head=args.head,
|
|
record_vetoes=args.experimental_record_vetoes,
|
|
declarations=args.experimental_declarations,
|
|
engine=args.engine,
|
|
background_deadline=args.background_deadline,
|
|
**options,
|
|
)
|
|
else:
|
|
# multiprocessing closes sys.stdin in its child; duplicate the Git pipe first.
|
|
with os.fdopen(os.dup(0), "rb") as stream:
|
|
if args.prior_hook:
|
|
from .model.contracts import ExecutionBudget
|
|
from .push.runner import _prepare
|
|
|
|
shared = ExecutionBudget(time.monotonic() + settings.deadline_seconds, settings.cancellation_allowance_seconds)
|
|
try:
|
|
prior_data = _prepare(lambda: stream.read(1024 * 1024 + 1), shared)
|
|
if len(prior_data) > 1024 * 1024:
|
|
raise ValueError("input exceeds hook integration limit")
|
|
except (ValueError, RuntimeError, TimeoutError):
|
|
print("Hook input could not be retained for all consumers. Retry with complete input and sufficient time.")
|
|
return 1
|
|
options["execution_budget"] = shared
|
|
|
|
def updates() -> str:
|
|
return (prior_data if prior_data is not None else stream.read(1024 * 1024 + 1)).decode("utf-8")
|
|
|
|
delivery = push(
|
|
args.path,
|
|
updates=updates,
|
|
remote_name=args.remote[0],
|
|
remote_url=args.remote[1],
|
|
engine=args.engine,
|
|
background_deadline=args.background_deadline,
|
|
**options,
|
|
)
|
|
except ValueError as error:
|
|
parser.error(str(error))
|
|
print(delivery.text, end="", flush=True)
|
|
if args.prior_hook and prior_data is not None:
|
|
# This is the explicitly chained hook's own behavior, outside analysis.
|
|
# Never replace its rejection with our advisory success or impose our timeout on it.
|
|
try:
|
|
prior = subprocess.run([str(args.prior_hook.resolve()), *args.remote], input=prior_data, cwd=args.path)
|
|
except OSError:
|
|
print("Existing hook could not be started. Restore its executable path before retrying.")
|
|
return 1
|
|
if prior.returncode:
|
|
return prior.returncode if prior.returncode > 0 else 128 - prior.returncode
|
|
return delivery.exit_code
|
|
|
|
|
|
def _scan(path: Path, config_path: Path, mode: str, fail_on: str) -> int:
|
|
outcome = _run_scan(path, config_path, mode, fail_on)
|
|
_print_scan_outcome(outcome)
|
|
return outcome.exit_code
|
|
|
|
|
|
def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOutcome:
|
|
if not path.exists() or not path.is_dir():
|
|
raise SystemExit(f"scan path is not a directory: {path}")
|
|
|
|
plan = plan_for_mode(mode)
|
|
config = load_config(config_path if config_path.exists() else None)
|
|
hunter_model = config.models.hunter
|
|
run = create_scan_run(path, config)
|
|
runtime = HarnessRuntime(
|
|
scan_id=run.scan_id,
|
|
config=config,
|
|
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets),
|
|
)
|
|
write_harness_config(config=config, path=run.root / "harness.json")
|
|
runtime.start(mode=mode, target=run.target)
|
|
policy = runtime.run_stage("policy_load", load_policy)
|
|
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
|
rule_ledger_path = run.target / CALIBRATION_DIR / "rule_policy.json"
|
|
ruleset = runtime.run_stage("ruleset_load", lambda: load_ruleset(DEFAULT_RULESET_DIR, rule_ledger_path))
|
|
runtime.run_stage("policy_check", lambda: assert_rules_resolve(ruleset, policy))
|
|
static_hints = runtime.run_stage("static_mapping", lambda: _load_static_hints(config.static_analysis.sarif_paths))
|
|
write_static_hints(static_hints, run.root / "mapping" / "static_hints.json")
|
|
snapshot, targets = runtime.run_stage(
|
|
"preprocess",
|
|
lambda: preprocess_repository(run.target, run.root / "preprocess" / "file_targets.json", static_hints),
|
|
)
|
|
# A language nothing here covers is named, never a silent zero (pre-push 17.2, Requirement 9.2).
|
|
for language, count in uncovered_languages(targets, ruleset, engine=mode != "quick"):
|
|
runtime.state["degradations"].append(
|
|
{
|
|
"stage": "quick_findings",
|
|
"reason": "language_not_covered",
|
|
"language": language,
|
|
"count": count,
|
|
"engine": "" if mode == "quick" else " and no engine frontend",
|
|
}
|
|
)
|
|
entry_points = runtime.run_stage("entry_point_mapping", lambda: analyze_entry_points(run.target, targets))
|
|
write_entry_points(entry_points, run.root / "mapping" / "entry_points.json")
|
|
targets = attach_reachability_hints(targets, entry_points)
|
|
write_preprocess_artifact(snapshot, targets, run.root / "preprocess" / "file_targets.json")
|
|
provenance = runtime.run_stage("provenance", lambda: fingerprint(run.target))
|
|
rankings = runtime.run_stage("rank", lambda: rank_targets(targets))
|
|
prior_learnings = load_false_positive_learnings(ledger_path)
|
|
rankings, calibrations = runtime.run_stage("calibrate", lambda: calibrate_rankings(rankings, prior_learnings))
|
|
applied_calibrations = [calibration for calibration in calibrations if calibration.applied_learning_ids]
|
|
write_rankings(rankings, run.root / "rank" / "ranking.json")
|
|
write_ranking_calibrations(calibrations, run.root / "calibration" / "applied_calibrations.json")
|
|
if mode == "standard":
|
|
hunter_result = runtime.run_stage(
|
|
"hunter_pool",
|
|
lambda: run_hunter_pool(run.target, targets, rankings, scan_id=run.scan_id, ruleset=ruleset, policy=policy),
|
|
)
|
|
findings = hunter_result.findings
|
|
trajectories = hunter_result.trajectories
|
|
else:
|
|
findings = runtime.run_stage(
|
|
"quick_findings",
|
|
lambda: quick_scan_findings(
|
|
run.target,
|
|
targets,
|
|
rankings,
|
|
ruleset,
|
|
policy,
|
|
min_emit_priority=config.ruleset.min_emit_priority,
|
|
min_emit_precision=config.ruleset.min_emit_precision,
|
|
),
|
|
)
|
|
trajectories = []
|
|
# Shadow-status rules fire but are excluded from the report and the score;
|
|
# their outcomes are preserved separately for precision tracking.
|
|
shadow_findings = [finding for finding in findings if finding.status == "shadow"]
|
|
if shadow_findings:
|
|
findings = [finding for finding in findings if finding.status != "shadow"]
|
|
write_findings(shadow_findings, run.root / "shadow_findings.json")
|
|
# Bounded-CI budget: cap the reported finding count (0 = unlimited). Findings are
|
|
# severity/priority-sorted, so truncation keeps the most severe and is disclosed.
|
|
max_findings = config.hardening.max_findings
|
|
if max_findings and len(findings) > max_findings:
|
|
runtime.state["degradations"].append(
|
|
{"stage": "budget", "reason": "max_findings_exceeded", "requested": max_findings, "actual": len(findings)}
|
|
)
|
|
findings = findings[:max_findings]
|
|
findings_path = run.root / "findings.json"
|
|
verification_path = run.root / "verification.json"
|
|
markdown_path = run.root / "report.md"
|
|
sarif_path = run.root / "report.sarif"
|
|
manifest_path = run.root / "manifest.json"
|
|
trajectories_path = run.root / "traces" / "hunter_trajectories.jsonl"
|
|
write_findings(findings, findings_path)
|
|
if trajectories:
|
|
write_hunter_trajectories(trajectories, trajectories_path)
|
|
verifications = runtime.run_stage("verify", lambda: verify_findings(findings))
|
|
write_verification_results(verifications, verification_path)
|
|
runtime.run_stage(
|
|
"record_calibration",
|
|
lambda: _persist_calibration_feedback(run, ledger_path, prior_learnings, findings, verifications),
|
|
)
|
|
# Fusion: deterministic two-panel adjudication for triggered findings (standard mode).
|
|
fusion_decisions: list[FusionDecision] = []
|
|
fusion_path = run.root / "fusion.json"
|
|
if mode == "standard" and config.fusion.enabled:
|
|
fusion_decisions = runtime.run_stage(
|
|
"fusion",
|
|
lambda: fuse_findings(findings, verifications, high_assurance=config.fusion.high_assurance),
|
|
)
|
|
if fusion_decisions:
|
|
fusion_path.write_text(json.dumps([decision.to_dict() for decision in fusion_decisions], indent=2, sort_keys=True) + "\n")
|
|
cwe_by_rule = {rule.rule_id: rule.cwe for rule in ruleset}
|
|
rule_cwe_by_id = {finding.finding_id: cwe_by_rule.get(finding.finding_id.split(":", 1)[0], "") for finding in findings}
|
|
# A confirmed false positive (prior-scan learning) lowers a finding's effective
|
|
# reachability multiplier in the score instead of deleting the rule.
|
|
reachability_override = fp_reachability_overrides(findings, prior_learnings)
|
|
score_artifact = runtime.run_stage(
|
|
"score",
|
|
lambda: build_score_artifact(
|
|
findings,
|
|
rule_cwe_by_id,
|
|
policy,
|
|
k=config.score.k,
|
|
min_score=config.score.min_score,
|
|
block_severity_reachable=config.score.block_severity_reachable,
|
|
blocking=config.score.blocking,
|
|
reachability_override=reachability_override,
|
|
),
|
|
)
|
|
score_path = run.root / "score.json"
|
|
score_path.write_text(json.dumps(score_artifact.to_dict(), indent=2, sort_keys=True) + "\n")
|
|
plan = record_completed(plan, Stage.STATIC)
|
|
complexity_payload: dict[str, object] | None = None
|
|
complexity_map_path = run.root / "complexity_map.json"
|
|
overlay_path = run.root / "overlay.json"
|
|
built_map = None
|
|
overlay_records: list[OverlayRecord] = []
|
|
wrote_overlay = False
|
|
obligations_payload: dict[str, object] | None = None
|
|
model_payload: dict[str, object] | None = None
|
|
obligations_cited: dict[str, dict[str, object]] = {}
|
|
if Stage.MAP in plan.requested:
|
|
built_map = runtime.run_stage(
|
|
"map",
|
|
lambda: complexity_map.build_complexity_map(
|
|
targets,
|
|
findings,
|
|
complexity_map_path,
|
|
repo_files=[target.path for target in targets],
|
|
ledger_path=run.target / CALIBRATION_DIR / "complexity_ledger.json",
|
|
),
|
|
)
|
|
facts_or_error: SemanticFacts | FactLoadError
|
|
try:
|
|
facts_or_error = load_facts()
|
|
except FactLoadError as exc:
|
|
facts_or_error = exc
|
|
runtime.state["degradations"].append({"stage": "overlay", "reason": "facts_unavailable", "detail": str(exc)})
|
|
overlay_records = list(
|
|
runtime.run_stage(
|
|
"overlay",
|
|
lambda: adjudicate(root=run.target, targets=targets, findings=findings, facts=facts_or_error),
|
|
)
|
|
)
|
|
write_overlay(overlay_records, overlay_path)
|
|
wrote_overlay = True
|
|
coverage_findings = [finding_from_coverage(record) for record in overlay_records if record.disposition == "coverage"]
|
|
if coverage_findings:
|
|
seen = {finding.finding_id for finding in findings}
|
|
extra = [item for item in coverage_findings if item.finding_id not in seen]
|
|
if extra:
|
|
findings = findings + extra
|
|
write_findings(findings, findings_path)
|
|
if config.obligations.enabled:
|
|
obligation_findings, obligations_payload, obligations_cited = _check_obligations(
|
|
root=run.target,
|
|
targets=targets,
|
|
entries=entry_points,
|
|
facts=facts_or_error,
|
|
settings=config.obligations,
|
|
runtime=runtime,
|
|
hunter_model=hunter_model,
|
|
)
|
|
if obligation_findings:
|
|
findings = rank_obligations(findings + obligation_findings)
|
|
write_findings(findings, findings_path)
|
|
# Fifth MAP proposer: the model layer (contributor-scan Req 1). Additive -- it adds findings beside
|
|
# the pattern, overlay and obligation ones and removes none of them.
|
|
model_findings, model_payload = _run_model_layer(
|
|
root=run.target,
|
|
targets=targets,
|
|
entries=entry_points,
|
|
settings=config.model,
|
|
runtime=runtime,
|
|
config=config,
|
|
)
|
|
if model_findings:
|
|
findings = findings + model_findings
|
|
write_findings(findings, findings_path)
|
|
plan = record_completed(plan, Stage.MAP)
|
|
if not hunter_model:
|
|
runtime.state["degradations"].append(skip_as_degradation(Stage.MAP, "hunter_model_unavailable"))
|
|
else:
|
|
built_map, findings, verifications = _attach_tool_hunter(
|
|
root=run.target,
|
|
built_map=built_map,
|
|
findings=findings,
|
|
verifications=verifications,
|
|
hunter_model=hunter_model,
|
|
max_hotspots=config.complexity.max_hunter_hotspots,
|
|
max_findings=max_findings,
|
|
findings_path=findings_path,
|
|
verification_path=verification_path,
|
|
complexity_map_path=complexity_map_path,
|
|
runtime=runtime,
|
|
)
|
|
complexity_payload = {
|
|
"hotspot_count": len(built_map.hotspots),
|
|
"heuristic_only": built_map.heuristic_only,
|
|
}
|
|
if Stage.REGRESS in plan.requested:
|
|
runtime.state["degradations"].append(skip_as_degradation(Stage.REGRESS, "regress_retired"))
|
|
plan = record_skip(plan, Stage.REGRESS, "regress_retired")
|
|
# Req 5.6: what each family could see, for the report. Absent when the model layer did not run --
|
|
# and a scan with no model layer states no limits because it made no claims.
|
|
coverage_rows = model_payload.get("coverage") if isinstance(model_payload, dict) else None
|
|
coverage_for_report = coverage_rows if isinstance(coverage_rows, list) and coverage_rows else None
|
|
runtime.run_stage(
|
|
"report",
|
|
lambda: write_markdown_report(
|
|
findings,
|
|
markdown_path,
|
|
verifications,
|
|
redact=config.hardening.redact_secrets,
|
|
complexity_map=built_map,
|
|
overlay=overlay_records if wrote_overlay else None,
|
|
obligations=obligations_cited or None,
|
|
obligations_summary=obligations_payload or None,
|
|
coverage=coverage_for_report,
|
|
# Degradations are coverage facts: an empty graph, a graph built in parts, a query that never
|
|
# ran. Kept out of the report they leave the reader unable to tell "nothing was found" from
|
|
# "nothing was looked at".
|
|
degradations=list(runtime.state.get("degradations") or ()),
|
|
),
|
|
)
|
|
runtime.run_stage(
|
|
"sarif",
|
|
lambda: write_sarif_report(
|
|
findings,
|
|
verifications,
|
|
sarif_path,
|
|
overlay=overlay_records if wrote_overlay else None,
|
|
obligations=obligations_cited or None,
|
|
),
|
|
)
|
|
runtime.run_stage(
|
|
"manifest",
|
|
lambda: write_manifest(
|
|
run=run,
|
|
findings=findings,
|
|
verifications=verifications,
|
|
artifact_paths=_artifact_paths(
|
|
findings=findings_path,
|
|
verification=verification_path,
|
|
markdown=markdown_path,
|
|
sarif=sarif_path,
|
|
score=score_path,
|
|
trajectories=trajectories_path if trajectories else None,
|
|
fusion=fusion_path if fusion_decisions else None,
|
|
complexity_map=complexity_map_path if complexity_payload is not None else None,
|
|
overlay=overlay_path if wrote_overlay else None,
|
|
),
|
|
path=manifest_path,
|
|
score=score_artifact.to_dict(),
|
|
degradations=runtime.state["degradations"] or None,
|
|
fusion=[_fusion_summary(decision) for decision in fusion_decisions] or None,
|
|
stages=stages_payload(plan),
|
|
complexity=complexity_payload,
|
|
provenance=provenance.to_dict(),
|
|
obligations=obligations_payload,
|
|
model=model_payload,
|
|
),
|
|
)
|
|
runtime.finish(status="succeeded")
|
|
return ScanOutcome(
|
|
scan_id=run.scan_id,
|
|
run_dir=run.root,
|
|
file_target_count=len(targets),
|
|
ranked_target_count=len(rankings),
|
|
finding_count=len(findings),
|
|
calibrations_applied=len(applied_calibrations),
|
|
exit_code=scan_exit_code(findings, verifications, fail_on),
|
|
notices=tuple(
|
|
f"language not covered: {item['language']} ({item['count']} files)"
|
|
for item in runtime.state["degradations"]
|
|
if item.get("reason") == "language_not_covered"
|
|
),
|
|
)
|
|
|
|
|
|
def _attach_tool_hunter(
|
|
*,
|
|
root: Path,
|
|
built_map: complexity_map.ComplexityMap,
|
|
findings: list[StaticFinding],
|
|
verifications: list[VerificationResult],
|
|
hunter_model: str,
|
|
max_hotspots: int,
|
|
max_findings: int,
|
|
findings_path: Path,
|
|
verification_path: Path,
|
|
complexity_map_path: Path,
|
|
runtime: HarnessRuntime,
|
|
) -> tuple[complexity_map.ComplexityMap, list[StaticFinding], list[VerificationResult]]:
|
|
client = tool_hunter.resolve_hunter_client()
|
|
if client is None:
|
|
return built_map, findings, verifications
|
|
hunter_hotspots = list(built_map.hotspots[: max(max_hotspots, 0)])
|
|
hunter_findings = runtime.run_stage(
|
|
"tool_hunter",
|
|
lambda: tool_hunter.run_tool_hunter(
|
|
root,
|
|
hunter_hotspots,
|
|
client=client,
|
|
model=hunter_model,
|
|
max_steps=tool_hunter.DEFAULT_MAX_STEPS,
|
|
),
|
|
)
|
|
seen = {finding.finding_id for finding in findings}
|
|
extra = [finding for finding in hunter_findings if finding.finding_id not in seen]
|
|
if extra:
|
|
findings = findings + extra
|
|
if max_findings and len(findings) > max_findings:
|
|
if not any(entry.get("reason") == "max_findings_exceeded" for entry in runtime.state["degradations"]):
|
|
runtime.state["degradations"].append(
|
|
{"stage": "budget", "reason": "max_findings_exceeded", "requested": max_findings, "actual": len(findings)}
|
|
)
|
|
findings = findings[:max_findings]
|
|
kept = {finding.finding_id for finding in findings}
|
|
extra = [finding for finding in extra if finding.finding_id in kept]
|
|
if extra:
|
|
verifications = verifications + verify_findings(extra)
|
|
write_verification_results(verifications, verification_path)
|
|
write_findings(findings, findings_path)
|
|
built_map = replace(built_map, heuristic_only=False)
|
|
complexity_map.write_complexity_map(built_map, complexity_map_path)
|
|
return built_map, findings, verifications
|
|
|
|
|
|
def _persist_calibration_feedback(
|
|
run: ScanRun,
|
|
ledger_path: Path,
|
|
prior_learnings: list[FalsePositiveLearning],
|
|
findings: list[StaticFinding],
|
|
verifications: list[VerificationResult],
|
|
) -> list[FalsePositiveLearning]:
|
|
# Store rejected or held findings as scoped learnings for the next scan.
|
|
new_learnings = learnings_from_verifications(findings, verifications)
|
|
merged = merge_false_positive_learnings(prior_learnings, new_learnings)
|
|
write_false_positive_learnings(merged, ledger_path)
|
|
write_false_positive_learnings(new_learnings, run.root / "calibration" / "false_positive_learnings.json")
|
|
return merged
|
|
|
|
|
|
def _print_scan_outcome(outcome: ScanOutcome) -> None:
|
|
print(f"scan_id={outcome.scan_id}")
|
|
print(f"run_dir={outcome.run_dir}")
|
|
print(f"file_targets={outcome.file_target_count}")
|
|
print(f"ranked_targets={outcome.ranked_target_count}")
|
|
print(f"findings={outcome.finding_count}")
|
|
print(f"calibrations_applied={outcome.calibrations_applied}")
|
|
for notice in outcome.notices:
|
|
print(f"notice: {notice}")
|
|
|
|
|
|
def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]:
|
|
hints: list[object] = []
|
|
for sarif_path in sarif_paths:
|
|
path = Path(sarif_path)
|
|
if path.exists():
|
|
hints.extend(ingest_sarif(path))
|
|
return hints
|
|
|
|
|
|
def _artifact_paths(**paths: Path | None) -> dict[str, Path]:
|
|
return {name: path for name, path in paths.items() if path is not None}
|
|
|
|
|
|
def _fusion_summary(decision: FusionDecision) -> dict[str, object]:
|
|
return {
|
|
"finding_id": decision.finding_id,
|
|
"disposition": str(decision.disposition),
|
|
"decision_source": decision.decision_source,
|
|
"triggers": decision.triggers,
|
|
}
|
|
|
|
|
|
def _benchmark(manifest_path: Path, config_path: Path, mode: str) -> int:
|
|
if not manifest_path.exists() or not manifest_path.is_file():
|
|
raise SystemExit(f"benchmark manifest is not a file: {manifest_path}")
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
target = resolve_benchmark_source(manifest_path, manifest)
|
|
run = create_benchmark_run(target, manifest)
|
|
started_at = perf_counter()
|
|
outcome = _run_scan(target, config_path, mode, "never")
|
|
runtime_seconds = perf_counter() - started_at
|
|
findings = _load_scan_findings(outcome.run_dir)
|
|
baseline_findings = load_baseline_findings(manifest_path, manifest)
|
|
result = evaluate_benchmark(
|
|
run=run,
|
|
mode=mode,
|
|
findings=findings,
|
|
scan_id=outcome.scan_id,
|
|
scan_run_dir=outcome.run_dir,
|
|
runtime_seconds=runtime_seconds,
|
|
baseline_findings=baseline_findings,
|
|
)
|
|
write_benchmark_artifacts(run, manifest_path, result)
|
|
_print_scan_outcome(outcome)
|
|
print(f"benchmark_run_id={run.benchmark_run_id}")
|
|
print(f"benchmark_run_dir={run.root}")
|
|
print(f"benchmark_expected={result.metrics.expected_findings_total}")
|
|
print(f"benchmark_matched={result.metrics.matched_findings_total}")
|
|
print(f"benchmark_missed={result.metrics.missed_findings_total}")
|
|
return 0
|
|
|
|
|
|
def _pairs(
|
|
catalog: Path,
|
|
slice_name: str,
|
|
*,
|
|
json_out: bool,
|
|
profile: str = "all",
|
|
split: str = "all",
|
|
pointers: bool = False,
|
|
) -> int:
|
|
if not catalog.exists() or not catalog.is_file():
|
|
raise SystemExit(f"pair catalog is not a file: {catalog}")
|
|
cases = select_split(select_profile(select_slice(load_pair_catalog(catalog), slice_name), profile), split)
|
|
try:
|
|
result = evaluate_catalog(cases, pointers=True if pointers else None)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"--k-runs: {exc}") from exc
|
|
if json_out:
|
|
print(json.dumps(result_payload(result), indent=2, sort_keys=True))
|
|
return 0
|
|
print_pair_metrics("pair eval", result)
|
|
print(f"signals={len(result.signals)}")
|
|
return 0
|
|
|
|
|
|
def _repos(directory: Path, name: str, *, fetch: bool, json_out: bool) -> int:
|
|
"""List the pinned checkouts and check each recipe against the code it claims to describe.
|
|
|
|
Offline by default: without ``--fetch`` a missing checkout is reported as missing, not fetched. Exit is
|
|
non-zero only when a materialised checkout CONTRADICTS its recipe -- a recipe that merely has not been
|
|
fetched is not a failure, or CI could never run this.
|
|
"""
|
|
recipes = select_repo(load_repo_recipes(directory), name)
|
|
if not recipes:
|
|
raise SystemExit(f"no repository recipe matches {name!r} in {directory}")
|
|
|
|
rows: list[dict[str, object]] = []
|
|
contradicted = 0
|
|
for recipe in recipes:
|
|
problems: tuple[str, ...] = ()
|
|
try:
|
|
root = resolve_repo(recipe, fetch=True if fetch else None)
|
|
except RepoUnavailable:
|
|
state, path = "absent", checkout_path(recipe)
|
|
else:
|
|
problems = verify_repo(recipe, root)
|
|
state, path = ("contradicted" if problems else "verified"), root
|
|
contradicted += 1 if problems else 0
|
|
rows.append({**recipe_payload(recipe), "state": state, "path": str(path), "problems": list(problems)})
|
|
|
|
if not json_out:
|
|
in_scope = len(recipe.in_scope)
|
|
measures = ",".join(recipe.measures)
|
|
print(f"{recipe.name:10} {state:13} {recipe.language:8} measures={measures} known={len(recipe.known)} in_scope={in_scope}")
|
|
for problem in problems:
|
|
print(f" ! {problem}")
|
|
if state == "absent":
|
|
print(f" fetch with: ousast repos --repo {recipe.name} --fetch")
|
|
|
|
if json_out:
|
|
print(json.dumps(rows, indent=2, sort_keys=True))
|
|
return 1 if contradicted else 0
|
|
|
|
|
|
def _model_skipped(reason: str) -> dict[str, object]:
|
|
"""A skipped stage reports the same SHAPE as a run one, plus why it skipped.
|
|
|
|
A consumer reading this should not have to branch on whether the stage ran; a payload that is sometimes
|
|
a full record and sometimes a single key is how a report ends up silently omitting a section.
|
|
"""
|
|
return {**_model_payload(None), "skipped": reason}
|
|
|
|
|
|
def _model_payload(
|
|
result: object, *, unjudged_paths: tuple[str, ...] = (), coverage: Sequence[Mapping[str, object]] = ()
|
|
) -> dict[str, object]:
|
|
"""What the scan records about the model layer.
|
|
|
|
`unjudged_sample` names the regions the budget did not reach, lowest-ranked first. Counting them is the
|
|
honesty requirement (Req 3.3); NAMING them is what makes an exclusion list for a large repository
|
|
something a maintainer can write from evidence rather than guesswork.
|
|
"""
|
|
scope = getattr(result, "scope", None)
|
|
outcomes = getattr(result, "question_outcomes", ())
|
|
if scope is not None:
|
|
unjudged_paths = tuple(
|
|
dict.fromkeys(
|
|
[q.identity.path for q in scope.deferred if q.reason != "tier_zero"]
|
|
+ [q.identity.path for q in outcomes if q.status != "completed"]
|
|
)
|
|
)[:10]
|
|
return {
|
|
"scope": scope.to_payload() if scope is not None else None,
|
|
"question_outcomes": [q.to_payload() for q in outcomes],
|
|
"family_coverage": [c.to_payload() for c in getattr(result, "family_coverage", ())],
|
|
"partitions": [asdict(p) for p in getattr(result, "partitions", ())],
|
|
"by_rung": dict(getattr(result, "by_rung", {}) or {}),
|
|
"regions_scanned": getattr(result, "regions_scanned", 0),
|
|
"regions_unjudged": getattr(result, "regions_unjudged", 0),
|
|
"regions_unasked": getattr(result, "regions_unasked", 0),
|
|
"unjudged_sample": list(unjudged_paths),
|
|
"model_calls": getattr(result, "model_calls", 0),
|
|
"cost_usd": getattr(result, "cost_usd", 0.0),
|
|
"seconds": getattr(result, "seconds", 0.0),
|
|
"build_seconds": getattr(result, "build_seconds", 0.0),
|
|
"query_seconds": getattr(result, "query_seconds", 0.0),
|
|
"query_seconds_by_kind": dict(getattr(result, "query_seconds_by_kind", {}) or {}),
|
|
"arbitrate_seconds": getattr(result, "arbitrate_seconds", 0.0),
|
|
"coverage": list(coverage),
|
|
}
|
|
|
|
|
|
def _run_model_layer(
|
|
*,
|
|
root: Path,
|
|
targets: list[FileTarget],
|
|
entries: Sequence[object],
|
|
settings: ModelLayerConfig,
|
|
runtime: HarnessRuntime,
|
|
config: ResolvedConfig,
|
|
) -> tuple[list[StaticFinding], dict[str, object]]:
|
|
"""Fifth MAP proposer (contributor-scan Req 1): the model layer, arbitrating regions of the repository.
|
|
|
|
Additive by construction. Without a CPG engine this records a reason and returns nothing, and the scan is
|
|
byte-for-byte what it is today -- the model layer must never make the tool worse for someone who does not
|
|
have Joern.
|
|
"""
|
|
if not settings.enabled:
|
|
return [], _model_skipped("disabled")
|
|
|
|
from .cpg.backend import resolve_cpg_backend
|
|
from .model.endpoint import resolve_chat_endpoint, resolve_models
|
|
from .model.regions import regions_for
|
|
from .model.scan import ScanBudget, scan_repository
|
|
from .model.shipped import declared_sources
|
|
|
|
backend = resolve_cpg_backend()
|
|
if not backend.available():
|
|
runtime.state["degradations"].append({"stage": "model", "reason": "cpg_unavailable"})
|
|
return [], _model_skipped("cpg_unavailable")
|
|
|
|
# The project's own build declaration, when it makes one, decides what the budget is spent on first.
|
|
regions = regions_for(entries, targets, shipped=declared_sources(root))
|
|
if not regions:
|
|
return [], _model_skipped("no_regions")
|
|
|
|
# Req 5.6: what each family could see, recorded whether or not it found anything. A family that ran
|
|
# and reported nothing is the case a reader is most likely to misread.
|
|
from .model.report import coverage_rows
|
|
|
|
coverage = coverage_rows(regions)
|
|
|
|
resolved = resolve_chat_endpoint(config)
|
|
client = resolved[0] if resolved else None
|
|
if client is None:
|
|
# Not a failure: entailment still works and only the suspicion band goes unasked (Req 11.3).
|
|
runtime.state["degradations"].append({"stage": "model", "reason": "learning_endpoint_unavailable"})
|
|
|
|
result = runtime.run_stage(
|
|
"model",
|
|
lambda: scan_repository(
|
|
root,
|
|
regions,
|
|
backend=backend,
|
|
client=client,
|
|
model=resolve_models(config)[0],
|
|
budget=ScanBudget(max_model_calls=settings.max_model_calls, max_regions=settings.max_regions),
|
|
),
|
|
)
|
|
for degradation in result.degradations:
|
|
runtime.state["degradations"].append(dict(degradation))
|
|
|
|
findings = [_finding_from_model(item) for item in result.findings]
|
|
return findings, _model_payload(result, coverage=coverage)
|
|
|
|
|
|
def _model_rationale(item: object, witness: str, family: str) -> str:
|
|
"""The witness, plus how many entry points reach this site when more than one does.
|
|
|
|
A site reached from twenty-six entry points is one defect with a wide blast radius, which is worth
|
|
saying once. It used to be said twenty-six times.
|
|
"""
|
|
base = witness or f"the model layer reported a {family} candidate"
|
|
reached = int(getattr(item, "reached_from", 1) or 1)
|
|
return base if reached <= 1 else f"{base} (reached from {reached} regions)"
|
|
|
|
|
|
def _finding_from_model(item: object) -> StaticFinding:
|
|
"""A model finding as a StaticFinding, carrying its rung and witness (Req 5.1)."""
|
|
site = str(getattr(item, "site", ""))
|
|
path, _, rest = site.partition(":")
|
|
line, _, function = rest.partition(":")
|
|
rung = getattr(item, "rung", None)
|
|
witness = str(getattr(item, "witness", "") or getattr(item, "contradiction", ""))
|
|
family = str(getattr(item, "family", ""))
|
|
return StaticFinding(
|
|
finding_id=f"model:{family}:{site}",
|
|
path=path or site,
|
|
title=f"{family} ({getattr(rung, 'value', rung)})",
|
|
severity="medium",
|
|
confidence="medium",
|
|
evidence_level="suspicion",
|
|
rationale=_model_rationale(item, witness, family),
|
|
line=int(line) if line.isdigit() else None,
|
|
function_name=function or None,
|
|
reachability_status="unknown",
|
|
reachability_evidence=[],
|
|
reachability_conditions=[],
|
|
tags=[f"family:{family}", f"rung:{getattr(rung, 'value', rung)}"],
|
|
ranking_priority=0.0,
|
|
rung=str(getattr(rung, "value", rung) or "suspicion"),
|
|
)
|
|
|
|
|
|
def _check_obligations(
|
|
*,
|
|
root: Path,
|
|
targets: list[FileTarget],
|
|
entries: Sequence[object],
|
|
facts: SemanticFacts | FactLoadError,
|
|
settings: ObligationsConfig,
|
|
runtime: HarnessRuntime,
|
|
hunter_model: str | None = None,
|
|
) -> tuple[list[StaticFinding], dict[str, object], dict[str, dict[str, object]]]:
|
|
"""Fourth MAP proposer (authorization-obligations Req 3-6): obligations without a dominating discharger, at suspicion.
|
|
|
|
Function-local until path records exist; the declared policy is read from the target; obligation findings never
|
|
enter the sandbox candidate set.
|
|
"""
|
|
from .semantic.ir import parse_file
|
|
from .semantic.obligations import load_obligation_facts
|
|
from .semantic.obligations.check import check_obligations, findings_to_static
|
|
from .semantic.obligations.dominance import OrderDominance
|
|
from .semantic.obligations.policy import PolicyError, load_declared_policy
|
|
|
|
payload: dict[str, object] = {"sibling_sets": 0, "under_populated": 0, "operations": 0, "findings_by_label": {}, "policy_version": None}
|
|
if isinstance(facts, FactLoadError):
|
|
runtime.state["degradations"].append({"stage": "obligations", "reason": "facts_unavailable"})
|
|
return [], payload, {}
|
|
irs: dict[str, tuple[Any, str]] = {}
|
|
texts: dict[str, str] = {}
|
|
for target in targets:
|
|
try:
|
|
text = (root / target.path).read_text(errors="ignore")
|
|
except OSError:
|
|
continue
|
|
texts[target.path] = text
|
|
irs[target.path] = (parse_file(target.path, text, target.language), text)
|
|
policy = None
|
|
policy_path = root / settings.policy_path
|
|
try:
|
|
policy = load_declared_policy(policy_path)
|
|
except PolicyError as exc:
|
|
runtime.state["degradations"].append({"stage": "obligations", "reason": "policy_invalid", "detail": str(exc)[:200]})
|
|
result = runtime.run_stage(
|
|
"obligations",
|
|
lambda: check_obligations(
|
|
irs=irs,
|
|
entries=entries,
|
|
facts=load_obligation_facts(),
|
|
flow_facts=facts,
|
|
policy=policy,
|
|
paths=(),
|
|
dominance=OrderDominance(texts=texts),
|
|
store_shapes=(), # known-fix hints came from the deleted mechanism store
|
|
min_siblings=settings.min_siblings,
|
|
),
|
|
)
|
|
from .semantic.obligations.intent import adjudicate_intent
|
|
|
|
result = adjudicate_intent(result, client=tool_hunter.resolve_hunter_client(), model=hunter_model or "", texts=texts)
|
|
for degradation in result.degradations:
|
|
runtime.state["degradations"].append(degradation)
|
|
by_label: dict[str, int] = {}
|
|
for finding in result.findings:
|
|
by_label[finding.label] = by_label.get(finding.label, 0) + 1
|
|
payload = {
|
|
"sibling_sets": len(result.sibling_sets),
|
|
"under_populated": len(result.under_populated),
|
|
"operations": result.operations,
|
|
"findings_by_label": by_label,
|
|
"policy_version": result.policy_version,
|
|
"degradations": [dict(item) for item in result.degradations],
|
|
"sets": [{"module": group.key[0], "resource": group.key[1], "handlers": len(group.handlers)} for group in result.sibling_sets],
|
|
}
|
|
statics = findings_to_static(result)
|
|
cited: dict[str, dict[str, object]] = {}
|
|
for finding, static in zip(result.findings, statics, strict=True):
|
|
cited[static.finding_id] = {
|
|
"obligation": finding.operation.kind,
|
|
"resource": finding.operation.resource,
|
|
"missing": finding.missing,
|
|
"provenance": finding.provenance,
|
|
"label": finding.label,
|
|
"evidence": list(finding.evidence),
|
|
"known_fix": finding.known_fix,
|
|
"intent": finding.intent,
|
|
"intent_rationale": finding.intent_rationale,
|
|
}
|
|
return statics, payload, cited
|
|
|
|
|
|
def _load_scan_findings(run_dir: Path) -> list[StaticFinding]:
|
|
return load_findings(run_dir / "findings.json")
|
|
|
|
|
|
def _learn_labels(args: argparse.Namespace) -> int:
|
|
"""``ousast learn labels``: exit 2 on a source outside the list, an excluded file or an unreadable clone."""
|
|
from .learn.labels import DEFAULT_CACHE, DEFAULT_SOURCES, LabelSourceError, build_labels, write_labels
|
|
|
|
try:
|
|
build, record = build_labels(
|
|
args.root, sources_path=args.sources or DEFAULT_SOURCES, cache=args.cache or DEFAULT_CACHE, include_title=args.include_title,
|
|
assumed_benign=not args.no_assumed_benign,
|
|
) # fmt: skip
|
|
except LabelSourceError as exc:
|
|
print(f"learn labels: {exc}", file=sys.stderr)
|
|
return 2
|
|
if args.out is not None:
|
|
digest = write_labels(build, args.out / "labels.jsonl")
|
|
(args.out / "labels.jsonl").rename(args.out / f"labels-{digest[:16]}.jsonl")
|
|
record["labels_sha256"] = digest
|
|
text = json.dumps(record, indent=2, sort_keys=True) + "\n"
|
|
if args.snapshot is not None:
|
|
args.snapshot.parent.mkdir(parents=True, exist_ok=True)
|
|
args.snapshot.write_text(text, encoding="utf-8")
|
|
pins = record["verified_pin_labels"]["by_family"]
|
|
print(f"labels: {record['rows']} rows in {record['groups']} repository groups; per family (positive/negative groups, model):")
|
|
for family, entry in pins.items():
|
|
print(f" {family}: {entry['positive_groups']}/{entry['negative_groups']} {entry['model']}")
|
|
return 0
|
|
|
|
|
|
def _plane(args: argparse.Namespace) -> int:
|
|
from .plane import reconciler # lazy: the plane is a maintainer surface, not the scan path
|
|
from .plane.profile import ProfileError, load_profile
|
|
|
|
try:
|
|
profile = load_profile(getattr(args, "profile", None))
|
|
except ProfileError as exc:
|
|
print(f"plane: {exc}", file=sys.stderr)
|
|
return 2
|
|
if args.plane_command == "run":
|
|
from .plane.memory import MemoryStoreError
|
|
|
|
_plane_memory("seed", args.run_manifest)
|
|
try:
|
|
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax, profile=profile)
|
|
except MemoryStoreError as exc: # the store is the state of record: no fallback, the Run is not submitted
|
|
print(f"memory store {profile.memory}: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(f"run finished: {result}")
|
|
_plane_memory("ingest", args.run_manifest)
|
|
return 0 if result == "done" else 1
|
|
if args.plane_command == "remember":
|
|
manifest = args.plane / "runs" / f"{args.run}.yaml"
|
|
return _plane_memory("ingest", manifest if manifest.is_file() else None, run=args.run, population=args.population, split=args.split)
|
|
if args.plane_command == "memory-normalise":
|
|
return _plane_memory_normalise(args)
|
|
if args.plane_command == "status":
|
|
print(reconciler.status(args.run, units=args.units))
|
|
return 0
|
|
if args.plane_command == "alerts-engine":
|
|
return _plane_alerts_engine(args)
|
|
if args.plane_command == "harvest":
|
|
return _plane_harvest(args, profile.atespace)
|
|
if args.plane_command == "doctor":
|
|
checks = reconciler.doctor(profile)
|
|
for name, ok, text in checks:
|
|
print(f"[{'ok' if ok else 'FAIL'}] {name}: {text}")
|
|
return 0 if all(ok for _, ok, _ in checks) else 1
|
|
if args.validation_set is not None:
|
|
return _plane_increment(args, profile.atespace)
|
|
from .plane.workspaces import workspaces
|
|
|
|
written = workspaces(args.population, args.out)
|
|
print(f"{len(written)} Workspace manifests written to {args.out}")
|
|
return 0
|
|
|
|
|
|
def _plane_harvest(args: argparse.Namespace, atespace: str) -> int:
|
|
from .plane.generate import repin_templates
|
|
from .plane.harvest import write_harvest
|
|
|
|
if args.runner_image is not None:
|
|
repin_templates(args.templates, args.runner_image)
|
|
command = f"ousast plane harvest --labels {args.labels.name} --name {args.name} --ceiling {args.ceiling:g}"
|
|
files = write_harvest(
|
|
args.labels, args.plane, name=args.name, command=command, catalog=args.catalog, cache=args.cache, templates=args.templates,
|
|
ceiling=args.ceiling, atespace=atespace,
|
|
) # fmt: skip
|
|
for relative in sorted(k for k in files if k.startswith("runs/")):
|
|
print(f"{args.plane / relative}: {files[relative].splitlines()[1].lstrip('# ')}")
|
|
index = json.loads(files["units.json"])
|
|
absent = sum(len(u.get("absent_on_side") or ()) for u in index.values())
|
|
print(f"absent_on_side: {absent} labelled candidates not declared on their side (not asked there)")
|
|
return 0
|
|
|
|
|
|
def _plane_alerts_engine(args: argparse.Namespace) -> int:
|
|
from .plane import engine_alerts
|
|
|
|
only = [c.strip() for c in args.cases.split(",") if c.strip()]
|
|
try:
|
|
results = engine_alerts.alerts_engine(
|
|
args.run_manifest, source=args.source, only=only, work=args.work, cache=args.repos or engine_alerts.CACHE,
|
|
deadline=args.deadline, image=args.image,
|
|
) # fmt: skip
|
|
except (engine_alerts.EngineAlertsError, OSError, ValueError, subprocess.CalledProcessError) as exc:
|
|
print(f"alerts-engine failed: {exc}", file=sys.stderr)
|
|
return 2
|
|
keys = ("case", "status", "reason", "alerts", "alerts_engine", "in_fix_range", "engine", "uncovered", "seconds")
|
|
for result in results:
|
|
print(json.dumps({k: result[k] for k in keys if k in result}, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
def _plane_memory_normalise(args: argparse.Namespace) -> int:
|
|
"""``ousast plane memory-normalise``: one pass over every row object; prints the counts."""
|
|
from .plane import memory
|
|
|
|
try:
|
|
store = memory.open_store(args.store)
|
|
got = store.normalise(dry_run=args.dry_run)
|
|
except (memory.MemoryStoreError, OSError, ValueError) as exc:
|
|
print(f"memory-normalise failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
verb = "would rewrite" if args.dry_run else "rewrote"
|
|
print(
|
|
f"memory {store.describe()}: {verb} {got['objects_rewritten']}/{got['objects']} objects; "
|
|
f'{got["rows_changed"]}/{got["rows"]} rows filled {got["fields_filled"]} absent or null fields with "" '
|
|
f"({', '.join(memory.QUERYABLE)})"
|
|
)
|
|
return 0
|
|
|
|
|
|
def _plane_memory(
|
|
step: str, run_manifest: Path | None, *, run: str | None = None, population: str | None = None, split: str | None = None
|
|
) -> int:
|
|
"""``seed`` before a Run, ``ingest`` after it (and ``ousast plane remember``). A store failure is reported and
|
|
never changes the run's result: ingest can be repeated by hand."""
|
|
from .plane import memory, reconciler
|
|
from .plane.tasks.remember import UNKNOWN, remember_run
|
|
|
|
try:
|
|
store = memory.open_store()
|
|
if step == "seed":
|
|
assert run_manifest is not None
|
|
seeded = memory.seed(run_manifest, store)
|
|
if seeded:
|
|
print(f"memory {store.describe()}: seeded {len(seeded)} tasks (reused facts): {', '.join(seeded)}")
|
|
return 0
|
|
annotations: dict[str, str] = {}
|
|
if run_manifest is not None:
|
|
spec, _ = reconciler.load_run(run_manifest)
|
|
run, annotations = run or spec.metadata.name, dict(spec.metadata.annotations)
|
|
assert run is not None
|
|
results = remember_run(
|
|
reconciler.run_dir(run), store, population=population or annotations.get(memory.POPULATION_ANNOTATION) or UNKNOWN,
|
|
split=split or annotations.get(memory.SPLIT_ANNOTATION) or UNKNOWN,
|
|
) # fmt: skip
|
|
except (memory.MemoryStoreError, OSError, ValueError) as exc:
|
|
print(f"memory {step} failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
fresh = [r for r in results if not r.skipped]
|
|
kinds = " ".join(f"{k}={n}" for k, n in memory.counts(fresh).items()) or "none"
|
|
print(f"memory {store.describe()}: {run}: {len(fresh)} tasks ingested, {len(results) - len(fresh)} already in the index; rows {kinds}")
|
|
return 0
|
|
|
|
|
|
def _plane_increment(args: argparse.Namespace, atespace: str) -> int:
|
|
from .plane.generate import increment
|
|
|
|
if args.candidates is None or args.triage is None:
|
|
print("--validation-set needs --candidates and --triage", file=sys.stderr)
|
|
return 2
|
|
home = str(Path.home())
|
|
paths = (args.population, args.validation_set, args.candidates, args.triage, args.repos)
|
|
shown = [str(p).replace(home, "~", 1) for p in paths]
|
|
command = (
|
|
f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]}"
|
|
f" --repos {shown[4]} --run {args.run}"
|
|
)
|
|
if args.runner_image is not None:
|
|
command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}"
|
|
written = increment(
|
|
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
|
|
repos=args.repos, runner_image=args.runner_image, atespace=atespace,
|
|
) # fmt: skip
|
|
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
|
|
return 0
|
|
|
|
|
|
def _model_candidates(args: argparse.Namespace) -> int:
|
|
"""What the candidate enumerator can reach, per family and per slice. No model is called anywhere here.
|
|
|
|
The ceiling is a property of the corpus and the enumerator, so it can end a design before a cent is spent
|
|
on it — which is what it did for the pattern-ruleset generator (12.1% against the IR's 96.6%).
|
|
"""
|
|
from .model.candidates import ceiling
|
|
from .model.taxonomy import load_families
|
|
|
|
cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice))
|
|
coverage = ceiling(cases, taxonomy=load_families(), generator=args.generator)
|
|
out: Path = args.out
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
payload = coverage.to_dict()
|
|
(out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
if args.json:
|
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
else:
|
|
for family, block in sorted(coverage.per_family.items()):
|
|
ceiling_value = float(block["ceiling"]) if isinstance(block["ceiling"], int | float) else 0.0
|
|
print(
|
|
f"model candidates {family}: {block['with_candidate']}/{block['pairs']} "
|
|
f"= {ceiling_value:.1%} ceiling, sites {block['sites']}"
|
|
)
|
|
for slice_name, block in sorted(coverage.per_slice.items()):
|
|
ceiling_value = float(block["ceiling"]) if isinstance(block["ceiling"], int | float) else 0.0
|
|
print(f" slice {slice_name}: {ceiling_value:.1%} over {block['pairs']} pairs")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|