marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-17 03:59:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pre-push-safety-net): enforce evidence eligibility and record blocked qualification

Browse Source
This commit is contained in:
Marius Ciepluch
2026-09-13 19:32:43 +02:00
parent 2491ce497c
commit b506516d87
14 changed files with 7394 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/push/README.md
+24
View File
@@ -188,3 +188,27 @@ reports nearest-rank p50/p95 with sample counts, and never treats no answers as
Exit 1 with a complete scorecard is measured NO-GO; a missing scorecard is an instrument
failure. Three samples per workload on development NodeGoat cannot independently qualify
a production language/framework/family or establish broad multi-project performance.
### Capability qualification (task 8.3)
The same `measure.py` replay instrument accepts `--selection development` (the
six PMPro/NodeGoat changes), `--selection regression` (three VAmPI targets), or
`--selection envelope` (libpng). Each profile retains the full declared population.
Readable regression snapshots may execute while their missing independent evaluation
prerequisites remain unresolved. `--deadline 600 --cache-dir /results/lab-cache`
is a separately frozen laboratory experiment, never a replacement for the 30-second
hook profile. A ranked or directly detected VAmPI target does not prove a transitive path.
`qualify_registry.py --evidence DIR --runtime SCORECARD --out NEW_FILE` recomputes
eligibility from the frozen profile, validated inputs, actual run and runtime scorecard.
Without `--declarations` it creates explicitly unreviewed, experimental declarations.
A reviewed declaration must match its frozen capability key, operation symbols, advice,
review identity, untouched workload and exact installed analysis semantics; all quality,
completion and runtime gates must pass. Synthetic positive controls in the unit suite
verify this contract and do not qualify real capabilities.
The installed `openultrasast/push/eligibility.json` is a trusted release input, not
repository configuration. The runner recomputes its decision and rejects stale or
modified evidence. This integrity check is not a cryptographic reviewer signature.
The versioned NO-GO registry enables no capability. Regenerate evidence after changes
to core, engine, facts, queries, policy or semantics; do not relabel old measurements.
benchmarks/push/measure.py
+31 -9
View File
@@ -163,31 +163,44 @@ def main() -> int:
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--cache", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--selection", choices=("development", "regression", "envelope"), default="development")
parser.add_argument("--deadline", type=float, default=30)
parser.add_argument("--cache-dir", type=Path)
args = parser.parse_args()
if args.deadline <= 0:
parser.error("deadline must be positive")
args.out.mkdir(parents=True, exist_ok=False)
manifest = json.loads(args.manifest.read_bytes())
validation = validate_manifest(args.manifest, cache_root=args.cache, fetch=False)
write(args.out / "input-validation.json", validation)
settings = ScanBudget(max_model_calls=0, max_regions=500, order_by_evidence=True)
initial = _provenance(args.manifest.parent, settings)
keys = ("ranking_mode", "engine", "facts", "queries", "policy", "core")
keys = ("ranking_mode", "engine", "engine_runtime", "facts", "queries", "policy", "core", "semantics")
config = {key: initial[key] for key in keys}
config.update(
version="first-replay-v2-2026-09-13",
version="qualification-replay-v1-" + args.selection,
scope="full tracked tree; declared first-party graph/target exclusions; evidence ranker",
hardware=platform.platform(),
budget_seconds=30,
budget_seconds=args.deadline,
targets=json.loads((args.manifest.parent / "diagnostic-config-v1.json").read_text())["targets"],
)
profile = freeze_profile(args.manifest.read_bytes(), config)
write(args.out / "profile.json", profile) # Freeze on disk BEFORE any measurement.
run = {"profile_sha256": profile["profile_sha256"], "context": dict(config), "records": []}
evidence = {"cases": [], "core_changes": [], "adapter_changes": [], "freeze_after_pmpro": None}
ready = {c["id"]: c["status"] == "ready" for c in validation["cases"]}
valid_snapshots = {s["id"] for s in validation["snapshots"] if s["status"] == "ready"}
ready = {
c["id"]: c["status"] == "ready" or (c["label"] == "regression" and c["base"] in valid_snapshots and c["tip"] in valid_snapshots)
for c in validation["cases"]
}
snapshots = {s["id"]: s for s in manifest["snapshots"]}
# Fixed declared order; reserved, regression and unsupported populations stay
# in the joint scorer without scanning or tuning on the holdout.
cases = [c for prefix in ("pmpro-", "nodegoat-") for c in manifest["cases"] if c["id"].startswith(prefix)]
if args.selection == "regression":
cases = [c for c in manifest["cases"] if c["label"] == "regression"]
elif args.selection == "envelope":
cases = [c for c in manifest["cases"] if c["label"] == "unsupported"]
with tempfile.TemporaryDirectory(prefix="ousast-experiment-") as temporary:
bare = Path(temporary) / "objects.git"
bare.mkdir()
@@ -209,6 +222,7 @@ def main() -> int:
args.out / "replay-pins.json",
{"snapshots": pins, "provenance": "Full upstream trees; declared authored edits in a private bare Git repository."},
)
seen_comparisons = set()
for case in cases:
if not ready[case["id"]]:
evidence["cases"].append({"case_id": case["id"], "status": "missing_input_prerequisite"})
@@ -231,14 +245,16 @@ def main() -> int:
"--artifact",
str(artifact_path),
"--deadline",
"30",
str(args.deadline),
"--cancellation-allowance",
"2",
"--max-regions",
"500",
]
if args.cache_dir is not None:
command += ["--cache-dir", str(args.cache_dir)]
start = time.monotonic()
process = subprocess.run(command, capture_output=True, text=True, timeout=62)
process = subprocess.run(command, capture_output=True, text=True, timeout=args.deadline + 5)
elapsed = time.monotonic() - start
(args.out / (case["id"] + ".stdout")).write_text(process.stdout)
(args.out / (case["id"] + ".stderr")).write_text(process.stderr)
@@ -257,7 +273,11 @@ def main() -> int:
# Unmeasured provenance caused by preparation timeout is explicit;
# never replace it with a fictitious runtime measurement.
entry["runtime_provenance_available"] = all(artifact["provenance"].get(key) == initial[key] for key in keys)
record = execution_record(case["id"], artifact, elapsed=elapsed, target=config["targets"][case["id"]])
record = execution_record(case["id"], artifact, elapsed=elapsed, target=config["targets"].get(case["id"]))
comparison = (pins[case["base"]], pins[case["tip"]])
if args.cache_dir is not None and comparison in seen_comparisons and artifact["timings"].get("graph_hits", 0) >= 2:
record["cache_state"] = "identical_tip"
seen_comparisons.add(comparison)
entry["coverage_reasons"] = artifact["admission"]["coverage_reasons"]
entry["stage_timings"] = artifact["timings"]
entry["snapshot_bytes_read"] = sum(s["bytes_read"] for s in artifact["snapshots"])
@@ -291,13 +311,15 @@ def main() -> int:
"decision": "NO-GO",
"capabilities_enabled": False,
"scorecard": "scorecard.json",
"reason": "First cold diagnostic experiment; independent eligibility and joint quality/warm-latency gates remain unmet.",
"reason": "Qualification evidence run; independent eligibility and joint quality/warm-latency gates remain unmet.",
"selection": args.selection,
"analysis_deadline_seconds": args.deadline,
"recorded_cases": len(run["records"]),
"declared_population": len(manifest["cases"]),
"core_unchanged": True,
"adapter_changes": [],
"limits": [
"No warm-cache measurements",
"No representative warm changed-code measurements; identical-comparison reuse is reported separately",
"Untouched Ghost population not scanned",
"PHP/Python admission populations missing",
"libpng retained as unsupported envelope; no C/C++ detection claim",
benchmarks/push/qualify_registry.py
+73
View File
@@ -0,0 +1,73 @@
"""Recompute versioned eligibility from frozen results; default declarations are experimental."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from openultrasast.model.scan import ScanBudget
from openultrasast.push.eligibility import create_decision
from openultrasast.push.policy import CapabilityAdmission, CapabilityKey
from openultrasast.push.runner import _provenance
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--evidence", type=Path, required=True)
parser.add_argument("--runtime", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--declarations", type=Path)
args = parser.parse_args()
def read(name):
return json.loads((args.evidence / name).read_bytes())
profile, validation, run = read("profile.json"), read("input-validation.json"), read("run.json")
current = _provenance(args.evidence, ScanBudget(max_model_calls=0, max_regions=500, order_by_evidence=True))
if args.declarations:
declarations = tuple(CapabilityAdmission.from_payload(d) for d in json.loads(args.declarations.read_bytes()))
else:
declarations = []
for capability in sorted({c["capability"] for c in profile["cases"]}):
language, framework, family = capability.split("/")
key = CapabilityKey(
language,
"unspecified",
framework,
family,
"dominance" if family == "access_control" else "taint",
"unreviewed",
current["semantics"],
)
declarations.append(
CapabilityAdmission(
key,
"unqualified-population-v1",
str(args.evidence / "scorecard.json"),
"experimental",
"Unreviewed consequence for {operation} in {context}.",
"Unreviewed repair for {operation} in {context}.",
)
)
decision = create_decision(
profile, validation, run, current=current, runtime=json.loads(args.runtime.read_bytes()), declarations=declarations
)
args.out.parent.mkdir(parents=True, exist_ok=True)
with args.out.open("x") as stream:
json.dump(decision, stream, indent=2, allow_nan=False)
stream.write("\n")
print(
json.dumps(
{
"verdict": decision["verdict"],
"enabled": sum(e["declaration"]["enabled"] for e in decision["entries"]),
"artifact_sha256": decision["artifact_sha256"],
}
)
)
return 0 if decision["verdict"] == "PASS" else 1
if __name__ == "__main__":
raise SystemExit(main())