mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 03:59:28 +00:00
feat(pre-push-safety-net): reuse compatible artifacts throughout replay
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
{
|
||||
"counts": {
|
||||
"load_bearing": 103,
|
||||
"load_bearing": 104,
|
||||
"standalone_capability": 9
|
||||
},
|
||||
"modules": 112,
|
||||
"note": "Refreshed for task 6.2: bounded local artifact storage, integrated by task 6.4.",
|
||||
"modules": 113,
|
||||
"note": "Refreshed for group 6: replay reaches compatible graph, query, discovery and comparison reuse.",
|
||||
"orphaned": [],
|
||||
"rows": [
|
||||
{
|
||||
@@ -118,6 +118,7 @@
|
||||
"config",
|
||||
"cpg.artifact",
|
||||
"model.contracts",
|
||||
"push.cache",
|
||||
"push.contracts",
|
||||
"push.policy"
|
||||
],
|
||||
@@ -139,6 +140,7 @@
|
||||
"model.taint",
|
||||
"push.cache",
|
||||
"push.policy",
|
||||
"push.reuse",
|
||||
"push.runner"
|
||||
],
|
||||
"module": "cpg",
|
||||
@@ -148,7 +150,10 @@
|
||||
"classification": "load_bearing",
|
||||
"importers": [
|
||||
"cpg.backend",
|
||||
"push.cache"
|
||||
"push.cache",
|
||||
"push.policy",
|
||||
"push.reuse",
|
||||
"push.runner"
|
||||
],
|
||||
"module": "cpg.artifact",
|
||||
"reason": "pre-push-safety-net Req 4.3/7.3: backend graph identity and live lease contract; validation integration pending"
|
||||
@@ -167,6 +172,7 @@
|
||||
"model.taint",
|
||||
"push.cache",
|
||||
"push.policy",
|
||||
"push.reuse",
|
||||
"push.runner"
|
||||
],
|
||||
"module": "cpg.backend",
|
||||
@@ -374,6 +380,7 @@
|
||||
"push.contracts",
|
||||
"push.policy",
|
||||
"push.report",
|
||||
"push.reuse",
|
||||
"push.runner",
|
||||
"push.snapshot"
|
||||
],
|
||||
@@ -421,6 +428,7 @@
|
||||
"push.contracts",
|
||||
"push.policy",
|
||||
"push.report",
|
||||
"push.reuse",
|
||||
"push.runner",
|
||||
"push.snapshot"
|
||||
],
|
||||
@@ -484,7 +492,8 @@
|
||||
"importers": [
|
||||
"model.partitions",
|
||||
"model.scan",
|
||||
"preprocess"
|
||||
"preprocess",
|
||||
"push.runner"
|
||||
],
|
||||
"module": "model.layout",
|
||||
"reason": "reachable from gate, map_gate, pair_gate or the CLI"
|
||||
@@ -513,6 +522,7 @@
|
||||
"model.layout",
|
||||
"model.scan",
|
||||
"push.policy",
|
||||
"push.reuse",
|
||||
"push.runner"
|
||||
],
|
||||
"module": "model.regions",
|
||||
@@ -690,6 +700,7 @@
|
||||
"cli",
|
||||
"push.policy",
|
||||
"push.report",
|
||||
"push.reuse",
|
||||
"push.runner",
|
||||
"push.snapshot"
|
||||
],
|
||||
@@ -698,7 +709,11 @@
|
||||
},
|
||||
{
|
||||
"classification": "load_bearing",
|
||||
"importers": [],
|
||||
"importers": [
|
||||
"push.policy",
|
||||
"push.reuse",
|
||||
"push.runner"
|
||||
],
|
||||
"module": "push.cache",
|
||||
"reason": "pre-push-safety-net task 6.2: bounded artifact publication and leased eviction; runner integration in 6.4"
|
||||
},
|
||||
@@ -707,6 +722,7 @@
|
||||
"importers": [
|
||||
"push.policy",
|
||||
"push.report",
|
||||
"push.reuse",
|
||||
"push.runner",
|
||||
"push.snapshot"
|
||||
],
|
||||
@@ -730,6 +746,14 @@
|
||||
"module": "push.report",
|
||||
"reason": "pre-push-safety-net Req 5.1/5.2/5.5/7.3: compact output and complete artifacts; runner integration pending"
|
||||
},
|
||||
{
|
||||
"classification": "load_bearing",
|
||||
"importers": [
|
||||
"push.runner"
|
||||
],
|
||||
"module": "push.reuse",
|
||||
"reason": "reachable from gate, map_gate, pair_gate or the CLI"
|
||||
},
|
||||
{
|
||||
"classification": "load_bearing",
|
||||
"importers": [
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@@ -137,3 +137,24 @@ flushed. Six one-shot runs are diagnostic samples, not representative warm p95 e
|
||||
Normal capabilities remain disabled, so zero alerts do not establish precision or useful
|
||||
recall. A separately named longer lab run may obtain witnesses, but cannot replace the
|
||||
30-second measurements or satisfy the hook's latency gate.
|
||||
|
||||
### Group 6 reuse correctness control
|
||||
|
||||
`reuse_control.py` creates an isolated, authored JavaScript history and invokes the packaged
|
||||
replay command for a cache-populating run, a 30-second identical-revision run, an uncached
|
||||
control, and dependency/one-function edits. It retains every artifact and checks equal evidence
|
||||
and current admission for compatible completed questions. This is development control data,
|
||||
not the untouched eligibility population or the representative warm p95 gate.
|
||||
|
||||
```sh
|
||||
docker run --rm --network none --memory 3g \
|
||||
-v /path/to/empty/results:/results \
|
||||
-v "$PWD/benchmarks/push/reuse_control.py:/control.py:ro" \
|
||||
--entrypoint python openultrasast:dev /control.py --out /results
|
||||
```
|
||||
|
||||
The output directory must be empty and writable by the image's nonroot user. Cold runs use a
|
||||
600-second laboratory budget; the identical-revision and changed-code controls retain the
|
||||
30-second deadline and 2-second cancellation allowance. Default normal capabilities remain
|
||||
empty. A passing reuse control cannot authorize rollout or turn unresolved coverage into clean
|
||||
analysis; groups 7–8 still own the real interface and joint evaluation.
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Packaged JavaScript reuse correctness control, not capability qualification.
|
||||
|
||||
Run with --out pointing to an empty directory outside any analyzed repository.
|
||||
Authored development code is never an independent precision/recall population.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
out = args.out.resolve()
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
if any(out.iterdir()):
|
||||
parser.error("output directory must be empty")
|
||||
repo = out / "repo"
|
||||
repo.mkdir()
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
git("init")
|
||||
git("config", "user.name", "Replay control")
|
||||
git("config", "user.email", "[email protected]")
|
||||
git("config", "core.hooksPath", "/dev/null")
|
||||
source = repo / "api.js"
|
||||
source.write_text('function handler(req, res) {\n const value = "fixed";\n eval(value);\n}\napp.post("/run", handler);\n')
|
||||
assert len(source.read_bytes()) > 0
|
||||
git("add", "api.js")
|
||||
git("commit", "-m", "fixed")
|
||||
base = git("rev-parse", "HEAD")
|
||||
source.write_text(source.read_text().replace('"fixed"', "req.body.value"))
|
||||
git("add", "api.js")
|
||||
git("commit", "-m", "introduce")
|
||||
head = git("rev-parse", "HEAD")
|
||||
runs = []
|
||||
|
||||
|
||||
def run(name, deadline, cache=True, tip=head):
|
||||
artifact = out / (name + ".json")
|
||||
cmd = [
|
||||
"python",
|
||||
"-m",
|
||||
"openultrasast.cli",
|
||||
"pre-push",
|
||||
str(repo),
|
||||
"--base",
|
||||
base,
|
||||
"--head",
|
||||
tip,
|
||||
"--artifact",
|
||||
str(artifact),
|
||||
"--deadline",
|
||||
str(deadline),
|
||||
"--max-regions",
|
||||
"20",
|
||||
]
|
||||
if cache:
|
||||
cmd += ["--cache-dir", str(out / "cache")]
|
||||
start = time.monotonic()
|
||||
process = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline + 10)
|
||||
elapsed = time.monotonic() - start
|
||||
(out / (name + ".stdout")).write_text(process.stdout)
|
||||
(out / (name + ".stderr")).write_text(process.stderr)
|
||||
assert artifact.is_file(), (process.returncode, process.stdout, process.stderr)
|
||||
data = json.loads(artifact.read_text())
|
||||
run = {
|
||||
"name": name,
|
||||
"elapsed_seconds": elapsed,
|
||||
"exit_code": process.returncode,
|
||||
"timings": data["timings"],
|
||||
"result": data["result"],
|
||||
"artifact": str(artifact),
|
||||
}
|
||||
runs.append(run)
|
||||
(out / "runs.json").write_text(json.dumps(runs, indent=2))
|
||||
print(json.dumps(run), flush=True)
|
||||
return data
|
||||
|
||||
|
||||
populated = run("populate", 600)
|
||||
warm = run("warm-identical", 30)
|
||||
cold = run("uncached", 600, False)
|
||||
(repo / "package.json").write_text('{"dependencies":{"express":"5.0.0"}}\n')
|
||||
git("add", "package.json")
|
||||
git("commit", "-m", "dependency change")
|
||||
changed = git("rev-parse", "HEAD")
|
||||
run("dependency-edit", 30, tip=changed)
|
||||
git("rm", "package.json")
|
||||
source.write_text(source.read_text().replace("req.body.value", 'req.body.value + "!"'))
|
||||
git("add", "api.js")
|
||||
git("commit", "-m", "one function edit")
|
||||
edited = git("rev-parse", "HEAD")
|
||||
run("one-function-edit", 30, tip=edited)
|
||||
|
||||
|
||||
def evidence(data):
|
||||
return [
|
||||
{
|
||||
"side": s["side"],
|
||||
"scope": s["scan"]["scope"],
|
||||
"questions": s["scan"]["question_outcomes"],
|
||||
"findings": s["scan"]["findings"],
|
||||
"degradations": s["scan"]["degradations"],
|
||||
}
|
||||
for s in data["scans"]
|
||||
]
|
||||
|
||||
|
||||
summary = {
|
||||
"cold_populated_evidence_equal": evidence(cold) == evidence(populated),
|
||||
"cold_warm_evidence_equal": evidence(cold) == evidence(warm),
|
||||
"admission_equal": cold["admission"] == warm["admission"],
|
||||
"runs": runs,
|
||||
"source_bytes": len(source.read_bytes()),
|
||||
"rollout": "not qualified; authored development control only",
|
||||
}
|
||||
(out / "summary.json").write_text(json.dumps(summary, indent=2))
|
||||
print(json.dumps(summary), flush=True)
|
||||
assert summary["cold_populated_evidence_equal"] and summary["cold_warm_evidence_equal"] and summary["admission_equal"]
|
||||
assert warm["timings"]["graph_hits"] == 2 and warm["timings"]["query_hits"] > 0
|
||||
assert any(item["candidate"]["delta"]["novelty"] == "new" for item in warm["admission"]["dispositions"])
|
||||
assert not any(data["admission"]["defects"] for data in (populated, warm, cold))
|
||||
assert runs[1]["elapsed_seconds"] <= 32
|
||||
assert all(
|
||||
r["timings"]["graph_hits"] <= 1 and r["elapsed_seconds"] <= 32 for r in runs if r["name"] in ("dependency-edit", "one-function-edit")
|
||||
)
|
||||
Reference in New Issue
Block a user