mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f5a904b7f2
commit
f63703a604
@@ -1,7 +1,7 @@
|
||||
# Reproducible push evaluation inputs
|
||||
|
||||
`inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run.
|
||||
Run `.venv/bin/python -m openultrasast.push_inputs benchmarks/push/inputs.json`.
|
||||
Run `.venv/bin/python -m benchmarks.push.push_inputs benchmarks/push/inputs.json`.
|
||||
It is offline by default, even if the ordinary repository network environment variable is set.
|
||||
`--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits.
|
||||
No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one
|
||||
|
||||
@@ -26,11 +26,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from measure import git, pin_snapshot # noqa: E402
|
||||
|
||||
from benchmarks.push.push_inputs import validate_manifest # noqa: E402
|
||||
from openultrasast.model.scan import ScanBudget # noqa: E402
|
||||
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
|
||||
from openultrasast.push.runner import _provenance # noqa: E402
|
||||
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
|
||||
from openultrasast.push_inputs import validate_manifest # noqa: E402
|
||||
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
|
||||
|
||||
TARGET = {"path": "app/routes/contributions.js", "function": "handleContributionsUpdate", "family": "injection"}
|
||||
|
||||
@@ -13,10 +13,10 @@ import time
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.push.push_inputs import validate_manifest
|
||||
from openultrasast.model.contracts import QuestionIdentity
|
||||
from openultrasast.model.scan import ScanBudget
|
||||
from openultrasast.push.runner import _provenance
|
||||
from openultrasast.push_inputs import validate_manifest
|
||||
from openultrasast.push_scoring import freeze_profile, score
|
||||
from openultrasast.repos import checkout_path, load_repo_recipe
|
||||
|
||||
@@ -314,7 +314,7 @@ def main() -> int:
|
||||
"reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.",
|
||||
"selection": args.selection,
|
||||
"analysis_deadline_seconds": args.deadline,
|
||||
"recorded_cases": len(run["records"]),
|
||||
"case_count": len(run["records"]),
|
||||
"declared_population": len(manifest["cases"]),
|
||||
"core_unchanged": True,
|
||||
"adapter_changes": [],
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Offline-first provenance checks for declared push evaluation populations.
|
||||
|
||||
This validates inputs, never detector outcomes or capability admission. Source bytes come
|
||||
from pinned Git blobs, so a dirty cached worktree cannot silently change the experiment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import replace
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from openultrasast.repos import RepoUnavailable, load_repo_recipe, resolve
|
||||
|
||||
|
||||
class InputError(ValueError):
|
||||
"""An input identity or label cannot be trusted."""
|
||||
|
||||
|
||||
def _blob(root: Path, commit: str, path: str) -> bytes:
|
||||
pure = PurePosixPath(path)
|
||||
if pure.is_absolute() or ".." in pure.parts or not path or "\\" in path:
|
||||
raise InputError(f"unsafe source path: {path!r}")
|
||||
# Git object reads neither follow worktree symlinks nor execute project code.
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "cat-file", "blob", f"{commit}:{path}"],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
if not result.stdout:
|
||||
raise InputError(f"empty source bytes: {path}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _snapshot(item: dict[str, Any], directory: Path, cache_root: Path | None, fetch: bool) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"id": item["id"], "status": "invalid_input", "bytes_read": 0, "files": []}
|
||||
try:
|
||||
recipe = load_repo_recipe(directory / item["recipe"])
|
||||
commit = item.get("commit", recipe.commit)
|
||||
if not isinstance(commit, str) or not re.fullmatch("[0-9a-f]{40}", commit):
|
||||
raise InputError("commit must be a full immutable SHA")
|
||||
recipe = replace(recipe, commit=commit)
|
||||
root = resolve(recipe, fetch=fetch, cache_root=cache_root)
|
||||
result.update(commit=commit, repository=recipe.url, license=recipe.license)
|
||||
files = item["files"]
|
||||
paths = [entry["path"] for entry in files]
|
||||
if recipe.license_file not in paths or len(paths) < 2 or len(paths) != len(set(paths)):
|
||||
raise InputError("declare a unique license file and at least one source file")
|
||||
for entry in files:
|
||||
content = _blob(root, commit, entry["path"])
|
||||
result["bytes_read"] += len(content)
|
||||
edit = entry.get("edit")
|
||||
if edit:
|
||||
if not isinstance(edit, dict):
|
||||
raise InputError("authored edit must be an object")
|
||||
if not edit.get("provenance", "").startswith("authored"):
|
||||
raise InputError("transformation must identify authored provenance")
|
||||
before, after = edit["before"].encode(), edit["after"].encode()
|
||||
if not before or content.count(before) != 1 or before == after:
|
||||
raise InputError(f"authored edit must match exactly once: {entry['path']}")
|
||||
if hashlib.sha256(content).hexdigest() != entry.get("original_sha256"):
|
||||
raise InputError(f"original hash mismatch: {entry['path']}")
|
||||
content = content.replace(before, after, 1)
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
if len(content) != entry["size"] or digest != entry["sha256"] or not content:
|
||||
raise InputError(f"source size/hash mismatch: {entry['path']}")
|
||||
result["files"].append({"path": entry["path"], "size": len(content), "sha256": digest})
|
||||
result["status"] = "ready"
|
||||
except RepoUnavailable:
|
||||
result.update(
|
||||
status="missing_prerequisite",
|
||||
problem=f"{recipe.name}: pinned checkout {commit} unavailable; rerun this validator with --fetch to fetch it",
|
||||
)
|
||||
except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as exc:
|
||||
result["problem"] = str(exc)
|
||||
return result
|
||||
|
||||
|
||||
def validate_manifest(path: Path, *, cache_root: Path | None = None, fetch: bool = False) -> dict[str, Any]:
|
||||
"""Retain every case, marking unavailable inputs and unreviewed labels explicitly.
|
||||
|
||||
`ready` means input provenance is verified. It never means a finding was reproduced.
|
||||
Explicit --fetch is required even if the ordinary repository network env var is set.
|
||||
"""
|
||||
manifest_bytes = path.read_bytes()
|
||||
data = json.loads(manifest_bytes)
|
||||
if not isinstance(data, dict) or data.get("schema_version") != 1:
|
||||
raise InputError("unsupported push input schema version")
|
||||
for field in ("snapshots", "cases"):
|
||||
if not isinstance(data.get(field), list) or not data[field]:
|
||||
raise InputError(f"{field} must be a nonempty list")
|
||||
for item in data[field]:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("id"), str) or not item["id"]:
|
||||
raise InputError(f"{field} entry requires an identity")
|
||||
snapshots = [_snapshot(item, path.parent, cache_root, fetch) for item in data["snapshots"]]
|
||||
indexed = {item["id"]: item for item in snapshots}
|
||||
if len(indexed) != len(snapshots):
|
||||
raise InputError("duplicate snapshot identity")
|
||||
cases = []
|
||||
ids: set[str] = set()
|
||||
for item in data["cases"]:
|
||||
case = dict(item, status="invalid_input", admission="experimental")
|
||||
try:
|
||||
if item["id"] in ids:
|
||||
raise InputError("duplicate case identity")
|
||||
ids.add(item["id"])
|
||||
if item["label"] not in {"vulnerable", "fixed", "benign", "regression", "unsupported", "unreviewed"}:
|
||||
raise InputError("invalid case label")
|
||||
if item["role"] not in {"development", "transfer", "regression", "envelope", "reserved_holdout"}:
|
||||
raise InputError("invalid workload role")
|
||||
if not item["capability"] or not item["provenance"]:
|
||||
raise InputError("capability and provenance are required")
|
||||
if item["label"] == "unreviewed" and not item["prerequisites"]:
|
||||
raise InputError("unreviewed labels require an explicit prerequisite")
|
||||
inputs = [indexed[item[key]] for key in ("base", "tip")]
|
||||
problems = [value.get("problem", "") for value in inputs if value["status"] != "ready"]
|
||||
if any(value["status"] == "invalid_input" for value in inputs):
|
||||
case["problem"] = "; ".join(problems)
|
||||
elif problems or item["prerequisites"]:
|
||||
case.update(status="missing_prerequisite", problem="; ".join(problems + item["prerequisites"]))
|
||||
else:
|
||||
case["status"] = "ready"
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
case["problem"] = str(exc)
|
||||
cases.append(case)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
|
||||
"population": len(cases),
|
||||
"capability_prerequisites": data.get("capability_prerequisites", {}),
|
||||
"snapshots": snapshots,
|
||||
"cases": cases,
|
||||
"note": "Input validation only; every capability remains experimental pending independent evaluation.",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("manifest", type=Path)
|
||||
parser.add_argument("--fetch", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = validate_manifest(args.manifest, fetch=args.fetch)
|
||||
except (ValueError, OSError) as exc:
|
||||
print(json.dumps({"status": "invalid_input", "problem": str(exc)}))
|
||||
return 2
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0 if all(case["status"] == "ready" for case in result["cases"]) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -31,11 +31,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from measure import git, pin_snapshot # noqa: E402
|
||||
|
||||
from benchmarks.push.push_inputs import validate_manifest # noqa: E402
|
||||
from openultrasast.model.scan import ScanBudget # noqa: E402
|
||||
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402
|
||||
from openultrasast.push.runner import _provenance # noqa: E402
|
||||
from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402
|
||||
from openultrasast.push_inputs import validate_manifest # noqa: E402
|
||||
from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402
|
||||
|
||||
# What each declared comparison is expected to show, stated before anything runs. `introduce`
|
||||
|
||||
Reference in New Issue
Block a user