mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
test(pre-push-safety-net): validate packaged outcome and retain rollout no-go
This commit is contained in:
@@ -201,3 +201,48 @@ Task 8.3 implementation verification: full suite **1317 passed, 9 skipped in 115
|
||||
14 focused eligibility/replay-instrument tests passed in 0.95s. Ruff, format, mypy
|
||||
(116 source files) and diff checks pass. These verify the delivered code and recorded
|
||||
NO-GO behavior, not the rejected full qualification claim above.
|
||||
|
||||
## Task 8.4 brief
|
||||
|
||||
Requirements 3.1, 3.4, 5.1–5.5, 6.5, 7.1, 7.3, 7.4, 8.3 and 8.4;
|
||||
packaging and full-feature validation boundary. The explicit task definition allows
|
||||
NO-GO validation while qualification remains blocked. Rebuild with installed eligibility,
|
||||
exercise the real local Git interface and final supported security/fixed/benign inputs,
|
||||
run canonical regression/static gates, and map all 35 acceptance criteria. No new
|
||||
production behavior; feature flag protocol is not applicable to validation artifacts.
|
||||
|
||||
The final image is `4cca6d20c5c9bfdbfa60c4e8fc991e096bc9b2f35096bb92cc92440e99f946d4`.
|
||||
The offline UID-1000 smoke imports and recomputes the installed resource as NO-GO,
|
||||
starts the CLI without model configuration, and exercises real local pushes in two
|
||||
disposable repositories. Prior exits 0 and 7 produce accepted/rejected Git pushes;
|
||||
Git's own rejection exit is 1. Exact stdin and remote arguments, source/index/HEAD,
|
||||
custom hooksPath, refused overwrite and byte-identical removal restoration all pass.
|
||||
The 0.1-second control truthfully reports unavailable coverage. It is not a detector
|
||||
positive control and does not install a hook into this project.
|
||||
|
||||
The final packaged six-case PMPro/Node replay also timed out in 30.560–30.614s,
|
||||
retaining all artifacts and input-byte receipts. No real alert or completed target is
|
||||
qualified. The instrument exits 1 with a completed NO-GO decision; the underlying
|
||||
advisory CLI exits 0, which means allow rather than complete or clean. The 38-file
|
||||
final verification bundle and all-35-criteria reconciliation are linked from
|
||||
`validation-group-8.md`.
|
||||
|
||||
## Status Report
|
||||
- STATUS: READY_FOR_REVIEW
|
||||
- TASK: 8.4
|
||||
- RED_PHASE_OUTPUT: N/A: validation harness and records, no production behavior change.
|
||||
- TESTS_RUN: Full suite 1317 passed, 9 skipped; current code unchanged since that run. Detection/map/local-pair gates exit 0. Ruff, format, mypy, compileall and diff checks pass. Final image builds and packaged hook smoke exits 0. Real security/fixed/benign replay completes with measured NO-GO.
|
||||
|
||||
## Review Verdict
|
||||
- VERDICT: APPROVED
|
||||
- TASK: 8.4
|
||||
- MECHANICAL_RESULTS: Actual validation harness and evidence inspected; all 35 original criterion numbers mapped, input reads verified, exact installed eligibility recomputed, prior hook behavior preserved. No new runtime imports or detector changes. Main-context Kiro fallback, not independent review.
|
||||
- FINDINGS: Task 8.4 explicitly requires an honest NO-GO artifact when joint gates fail. Task 8.3 remains unchecked and blocked; feature phase stays implementation. No threshold, label, population or ownership workaround used to claim rollout.
|
||||
- SUMMARY: Packaged acceptance validation delivered with reproducible NO-GO and concrete owner follow-up.
|
||||
|
||||
## Verification Result
|
||||
- STATUS: VERIFIED
|
||||
- CLAIM_TYPE: TASK
|
||||
- CLAIM: Task 8.4 validation and acceptance reconciliation completed with NO-GO.
|
||||
- EVIDENCE: `validation-group-8.md`, versioned packaged validation JSON and verified 38-file bundle; all commands and limitations recorded there.
|
||||
- GAPS: Feature-level readiness is not verified. Task 8.3 and its owner/evaluation prerequisites remain open.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"feature_name": "pre-push-safety-net",
|
||||
"created_at": "2026-09-12T15:35:00Z",
|
||||
"updated_at": "2026-09-13T16:41:21.889949+00:00",
|
||||
"updated_at": "2026-09-13T17:38:57.553583+00:00",
|
||||
"language": "en",
|
||||
"phase": "implementation",
|
||||
"approvals": {
|
||||
@@ -19,7 +19,7 @@
|
||||
}
|
||||
},
|
||||
"ready_for_implementation": true,
|
||||
"implementation_approval_note": "Group 8 authorized and in progress; groups 1\u20137 verified. Capability admission and rollout remain NO-GO until the declared evidence passes.",
|
||||
"implementation_approval_note": "Group 8 authorized: 8.1, 8.2 and 8.4 verified; 8.3 implementation/evidence delivered but qualification blocked. Feature rollout remains NO-GO, no capabilities enabled.",
|
||||
"depends_on": [
|
||||
"contributor-scan"
|
||||
],
|
||||
@@ -117,9 +117,10 @@
|
||||
"7.2",
|
||||
"7.3",
|
||||
"8.1",
|
||||
"8.2"
|
||||
"8.2",
|
||||
"8.4"
|
||||
],
|
||||
"remaining_tasks": 2,
|
||||
"remaining_tasks": 1,
|
||||
"verification_record": "implementation-group-8.md",
|
||||
"verification_records": [
|
||||
"implementation-group-1.md",
|
||||
@@ -136,6 +137,12 @@
|
||||
"8.3"
|
||||
],
|
||||
"feasibility": "NO-GO: all 21 frozen NodeGoat transactions timed out; 0/24 declared target checks completed. Cancellation/reporting remained below 2 seconds. Priming did not qualify graph reuse; named census and engine lifecycle work routed to contributor-scan.",
|
||||
"qualification": "NO-GO: eligibility implementation and evidence delivered; required independent populations and verified VAmPI transitive witness remain missing. Zero capabilities enabled."
|
||||
"qualification": "NO-GO: eligibility implementation and evidence delivered; required independent populations and verified VAmPI transitive witness remain missing. Zero capabilities enabled.",
|
||||
"validation": {
|
||||
"decision": "NO-GO",
|
||||
"feature_complete": false,
|
||||
"record": "validation-group-8.md",
|
||||
"capabilities_enabled": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,7 +235,7 @@ independent capability qualification. Groups 7–8 remain pending; rollout remai
|
||||
- _Depends: 1.2, 1.3, 5.2, 7.3, 8.2_
|
||||
- _Requirements: 3.1, 3.2, 6.1, 6.2, 6.3, 6.4, 6.5, 8.1, 8.2, 8.3, 8.4_
|
||||
- _Blocked 2026-09-13:_ Eligibility implementation and reproducible NO-GO artifact delivered; independent reviewed Node/PHP/Python populations and verified VAmPI transitive witness remain missing. Runtime/coverage gates fail. See `implementation-group-8.md` and contributor-scan owner follow-up.
|
||||
- [ ] 8.4 Verify the packaged end-to-end outcome and acceptance coverage
|
||||
- [x] 8.4 Verify the packaged end-to-end outcome and acceptance coverage
|
||||
- Run the required regression checks and a local packaged smoke over supported security/fixed/benign pushes; verify no-model startup, preservation of hook behavior and truthful partial results.
|
||||
- Reconcile every acceptance criterion with fresh evidence and record unresolved capability/performance blockers separately from completed plumbing.
|
||||
- Done when a reproducible validation artifact states GO only with all mandatory integration and joint quality/latency gates satisfied; otherwise it names the remaining NO-GO conditions without marking the feature complete.
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
# Packaged acceptance validation — group 8
|
||||
|
||||
## Validation Report
|
||||
- DECISION: NO-GO
|
||||
- FEATURE_COMPLETE: false
|
||||
- BLOCKED_TASKS: 8.3 — independent reviewed populations and verified VAmPI transitive evidence are missing; useful completion and runtime gates fail.
|
||||
- OWNERSHIP: UPSTREAM for engine/census/context prerequisites; LOCAL for evaluation population and capability qualification.
|
||||
- UPSTREAM_SPEC: contributor-scan; see `pre-push-runtime-followup.md` there.
|
||||
|
||||
This report distinguishes implemented safety contracts from demonstrated useful detection.
|
||||
All 35 criteria below are mapped. A contract test does not qualify a language or framework,
|
||||
and this report makes no rollout or production-readiness claim. Task 8.4 permits this
|
||||
explicit NO-GO outcome; task 8.3 and the feature remain incomplete.
|
||||
|
||||
## Acceptance coverage
|
||||
|
||||
Paths in the evidence column refer to `tests/` unless otherwise indicated. Current
|
||||
full-suite evidence covers the code at the group 8.3 implementation commit. Packaged
|
||||
receipts below exercise the built release with its installed NO-GO registry.
|
||||
|
||||
| Criterion | Evidence | Assessment |
|
||||
|---|---|---|
|
||||
| 1.1 | `test_push_snapshot.py`, `test_push_interface.py`, `test_push_regression_integration.py` | Exact immutable comparisons/ref associations verified by controls. |
|
||||
| 1.2 | Snapshot/integration tests; packaged disposable-hook smoke | Working source/index/refs and preceding hook preserved. |
|
||||
| 1.3 | Snapshot and multi-ref tests | Delete/missing-base/unsupported dispositions remain explicit. |
|
||||
| 1.4 | Snapshot/context tests | Comparison rules, force/new-branch/tag/rename identities retained. |
|
||||
| 2.1 | Context/scope integration tests; real Node/VAmPI artifacts | Ranker owns selection; useful context incomplete on real workloads. |
|
||||
| 2.2 | Scope, runner and failed-census controls | Omitted context is an explicit gap; real dynamic dependencies remain unresolved. |
|
||||
| 2.3 | `test_push_regression_integration.py`, partition/backend tests | Physical first-party filtering verified; external semantics remain unresolved. |
|
||||
| 2.4 | Removed-guard context/admission tests | Contract verified; real dominance context projection remains unqualified. |
|
||||
| 3.1 | `test_push_admission.py`, `test_push_report.py`, `test_push_eligibility.py` | Actionability admission enforced; zero qualified real capabilities. |
|
||||
| 3.2 | Admission/eligibility tests; installed NO-GO registry | Unreviewed/unsupported candidates do not become normal alerts. |
|
||||
| 3.3 | Admission/report tests | Duplicate witness paths produce one defect summary. |
|
||||
| 3.4 | Delta/admission/report tests; real same-snapshot VAmPI | Unchanged or unknown novelty cannot create a new-defect alert. |
|
||||
| 3.5 | Model-assistance tests | Model selection cannot create evidence or change admission. |
|
||||
| 4.1 | Execution-budget and multi-ref tests; real timeout receipts | Single absolute budget retained; useful completion gate fails. |
|
||||
| 4.2 | Cancellation/publication controls and 21-case runtime profile | Partial artifacts retained; measured cancellation/report allowance below 2s. |
|
||||
| 4.3 | Cache invalidation/reuse tests and packaged group 6 control | Exact input/context/semantic compatibility required. |
|
||||
| 4.4 | Corrupt/partial/cache-failure controls | No partial or stale clean verdict reused. |
|
||||
| 5.1 | Report/CLI tests and packaged output | Compact supported-scope status; no repository-security claim. |
|
||||
| 5.2 | Timeout artifacts and report tests | Compact operational notice with next action/artifact, no per-file warning flood. |
|
||||
| 5.3 | CLI/admission tests; packaged advisory/prior-hook smoke | Advisory permits push, preserves preceding rejection, same evidence bar. |
|
||||
| 5.4 | Multi-ref/CLI policy tests | Blocking and incomplete-coverage policies separate; real blocking qualification absent. |
|
||||
| 5.5 | Result contracts, artifact tests and real timeout artifacts | Finding/coverage/push disposition remain separate. |
|
||||
| 6.1 | Input validator and frozen profiles | BLOCKED: independent Node/PHP/Python reviewed populations missing. |
|
||||
| 6.2 | Versioned runtime/qualification scorecards | Counts, intervals, undefined precision, unresolved cases and latency retained; gates fail. |
|
||||
| 6.3 | Scorer controls and real VAmPI replay | Missing target/question/transitive proof remains unresolved in denominator. |
|
||||
| 6.4 | Frozen provenance receipts, deployed evidence ranking mode | No comparative ranker retune claimed; fixed core preserved across PHP/Node. |
|
||||
| 6.5 | Eligibility recomputation and NO-GO decisions | Joint gates enforced; no runtime/quality qualification earned. |
|
||||
| 7.1 | Packaged network-disabled no-model runs | No endpoint, engine download, source fetch or project build required by hook. |
|
||||
| 7.2 | Explicit/delayed/misleading-model controls | Redaction, model provenance and shared budget verified on controlled path. |
|
||||
| 7.3 | Full raw bundles, graph/query cache identity tests | Input, selected/deferred questions, outcomes, exclusions and stage identities retained. |
|
||||
| 7.4 | Hook tests; packaged installer/refusal/removal smoke | Explicit integration preserves hooks and configuration. |
|
||||
| 8.1 | Equivalent-normalized-evidence ranking/admission controls | Shared core rules; no PHP/Node priority exceptions introduced. |
|
||||
| 8.2 | Runtime smoke and missing-census controls; real Node/VAmPI gaps | Missing frontend/context evidence is not interpreted as safe. |
|
||||
| 8.3 | Frozen six-case packaged PHP/Node profiles | Transfer attempted unchanged; all 30s cases time out. Useful transfer unproven. |
|
||||
| 8.4 | Separate development/regression/envelope profiles | No core/fact/adapter retune; libpng arithmetic/bounds unsupported; no C++ detection claim. |
|
||||
|
||||
## Integration and boundary assessment
|
||||
|
||||
The implemented route remains Snapshot Adapter → ranker-owned scope → existing scan
|
||||
engine/cache → delta/actionability admission → report/hook. Persistent eligibility
|
||||
is a trusted installed input to the actionability boundary. It recomputes the existing
|
||||
evaluation scorer and checks exact semantics; repository configuration and optional
|
||||
models cannot supply eligibility. Immutable graph leases and answer compatibility are
|
||||
kept separate from change attribution. No incremental graph mutation, second engine,
|
||||
new IR, rank boost or downstream detector workaround was introduced in group 8.
|
||||
|
||||
Concrete remaining gaps are engine lifecycle/census completeness and context/witness
|
||||
projection, not evidence that a new ranker would solve the measured failure. Shared
|
||||
scan changes belong to contributor-scan and must trigger downstream replay revalidation.
|
||||
The file structure extends the approved evaluation/admission boundary with
|
||||
`push/eligibility.py` and its installed JSON resource. Ordinary scan behavior remains
|
||||
covered by the full suite and canonical detector/map/pair gates.
|
||||
|
||||
## Required next work
|
||||
|
||||
1. In contributor-scan, reconcile named readable first-party files with the Node graph
|
||||
census, then measure/amortize frontend, overlay, census and query process startup.
|
||||
Preserve vendor exclusion and complete/corrupt cache distinctions.
|
||||
2. Resolve supported first-party context and retain actual query-origin/path witnesses.
|
||||
Reproduce the three VAmPI targets with exact question accounting; a raw SQLI finding
|
||||
does not establish transitive reachability or push attribution.
|
||||
3. Freeze independently reviewed Node API vulnerable/fixed/benign cases with the required
|
||||
middleware/module/async shapes; obtain separate PHP/Python qualification populations.
|
||||
Do not tune on reserved Ghost before its population is fixed.
|
||||
4. Refreeze the candidate after owner fixes and rerun all seven runtime workload classes,
|
||||
cross-language regressions and independent actionable-alert review. Require the
|
||||
unchanged 30s warm p95, 2s cancellation allowance, 95% completion/observed precision,
|
||||
90% supported recall and zero known fixed/benign false alerts together.
|
||||
5. Regenerate eligibility only for exact passing language/framework/family semantics,
|
||||
then rerun packaged validation. Keep normal capabilities disabled until then.
|
||||
|
||||
## Fresh verification evidence
|
||||
|
||||
- Full suite: `pytest -q`, **1317 passed, 9 skipped**, 115.80s, exit 0. No production
|
||||
source changed after this run; subsequent edits are validation tooling and records.
|
||||
- Ruff check and formatting pass (262 files); mypy passes (116 source files);
|
||||
compileall exits 0. No new placeholder markers or concrete secrets in reviewed code.
|
||||
- Detection regression gate: 44/47 recall, 0/44 false positives, PASS. Map gate: PASS.
|
||||
Local pair gate: 3/3 pair-correct, 28/28 labeled recall, no fixed-side leaks, PASS.
|
||||
These corpus controls do not qualify real pre-push language/framework capabilities.
|
||||
- Docker build exits 0, image
|
||||
`4cca6d20c5c9bfdbfa60c4e8fc991e096bc9b2f35096bb92cc92440e99f946d4`.
|
||||
- Packaged offline smoke exits 0 as UID 1000. Installed registry recomputes as NO-GO,
|
||||
zero enabled capabilities. Both real disposable hook acceptance/rejection and removal
|
||||
controls pass; exact stdin/remote args and prior state are verified.
|
||||
- Final packaged six-case PHP/Node replay completes its instrument with exit 1 and
|
||||
an explicit NO-GO decision. All underlying advisory CLI invocations exit 0 and save
|
||||
partial artifacts; all six time out in 30.560–30.614s. Every snapshot has positive
|
||||
byte receipts. No target check or real actionable alert is qualified. Preparation
|
||||
timeout provenance remains unavailable where it was not measured.
|
||||
- Versioned record: `benchmarks/measurements/2026-09-13-push-packaged-validation.json`.
|
||||
Its 38-file raw bundle is 827,483 bytes, SHA-256
|
||||
`ad1bc15791636868a65a5cbb93b84d1a08dc6c94be0eb2f8f03890936720cc79`.
|
||||
Decompression matches every retained original. Full logs, smoke artifacts, profile,
|
||||
replay pins, scorecard and six complete partial-result artifacts are included.
|
||||
|
||||
## Verification Result
|
||||
- STATUS: NOT_VERIFIED
|
||||
- CLAIM_TYPE: FEATURE_GO
|
||||
- CLAIM: Pre-push safety net ready for rollout.
|
||||
- EVIDENCE: Passing integration controls coexist with measured runtime/completion failures and missing quality prerequisites.
|
||||
- GAPS: Task 8.3 qualification; real independent positive/fixed/benign evidence; verified VAmPI transitive witness; useful changed-code latency and coverage.
|
||||
- NOTES: Keep the feature in implementation and capabilities disabled. Follow owner remediation above, then rerun qualification and packaged validation.
|
||||
@@ -91,11 +91,23 @@ the deadline in preparation/discovery before graph analysis, so compatible prepa
|
||||
reuse matters alongside graph persistence. Group 6 now provides compatible local reuse; warm changed-code performance
|
||||
and useful coverage remain mandatory later gates. No ranking/fact adaptation or holdout
|
||||
inspection occurred in the final transfer run. Real capability eligibility and automatic
|
||||
dominance context projection remain unproven; the default capability registry is empty.
|
||||
dominance context projection remain unproven; the installed capability registry is NO-GO with no enabled capabilities.
|
||||
|
||||
Group 6 verified equal cached/uncached evidence and current admission in a packaged authored
|
||||
JavaScript control. Source/declaration edits invalidate affected units. Identical-revision
|
||||
latency does not qualify representative changed-code performance. Requirements and rollout
|
||||
thresholds remain unchanged; group 8 and independent capability admission remain pending. Group 7 adds exact multi-ref
|
||||
thresholds remain unchanged; independent capability admission remains blocked in group 8. Group 7 adds exact multi-ref
|
||||
transactions, explicit non-overwriting hook integration and bounded optional witness
|
||||
selection. No model output can create evidence, raise a rung or bypass admission.
|
||||
|
||||
|
||||
Group 8 validation (2026-09-13) is **NO-GO**. Tasks 8.1, 8.2 and 8.4 are verified;
|
||||
26/27 tasks are complete. Task 8.3 remains blocked despite delivered eligibility
|
||||
implementation and a reproducible installed NO-GO resource. All 21 representative
|
||||
Node runtime transactions timed out (0/24 declared target checks complete); final
|
||||
packaged PHP/Node security/fixed/benign replays likewise time out. VAmPI preserves
|
||||
raw known findings and valid identical-comparison reuse, but exact target question
|
||||
context/transitive evidence remains unresolved. Independent reviewed Node/PHP/Python
|
||||
populations are missing; libpng arithmetic/bounds remains unsupported. No real capability
|
||||
is enabled. Engine/census/context prerequisites route to contributor-scan before another
|
||||
frozen qualification attempt. See `pre-push-safety-net/validation-group-8.md`.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@@ -212,3 +212,23 @@ repository configuration. The runner recomputes its decision and rejects stale o
|
||||
modified evidence. This integrity check is not a cryptographic reviewer signature.
|
||||
The versioned NO-GO registry enables no capability. Regenerate evidence after changes
|
||||
to core, engine, facts, queries, policy or semantics; do not relabel old measurements.
|
||||
|
||||
### Packaged acceptance smoke (task 8.4)
|
||||
|
||||
After rebuilding the image with its installed eligibility resource, run:
|
||||
|
||||
```sh
|
||||
docker run --rm --network none --entrypoint python \
|
||||
-v "$PWD:/context:ro" -v /absolute/writable/results:/results \
|
||||
openultrasast:dev /context/benchmarks/push/package_smoke.py \
|
||||
--out /results/new-package-smoke --ops /context/ops
|
||||
```
|
||||
|
||||
This smoke verifies packaged CLI startup, the installed reproducible NO-GO registry,
|
||||
non-overwriting installation, exact prior-hook stdin/remote arguments, accept/reject
|
||||
behavior and explicit removal in disposable Git repositories. Its 0.1-second deadline
|
||||
intentionally exercises unavailable coverage, not vulnerability detection. No hook is
|
||||
installed in this project. Run `measure.py` separately on the final image for the full
|
||||
supported security/fixed/benign workload, preserving its real partial outcomes.
|
||||
The artifact and `.kiro/specs/pre-push-safety-net/validation-group-8.md` distinguish
|
||||
working integration from the remaining NO-GO quality/runtime prerequisites.
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Offline packaged CLI/registry and disposable Git hook validation, not detector qualification."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.model.scan import ScanBudget
|
||||
from openultrasast.push.eligibility import _DEFAULT, load_registry
|
||||
from openultrasast.push.runner import _provenance
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--ops", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
out = args.out
|
||||
out.mkdir(exist_ok=False)
|
||||
|
||||
|
||||
def run(argv, **kwargs):
|
||||
return subprocess.run(argv, capture_output=True, timeout=20, **kwargs)
|
||||
|
||||
|
||||
help_result = run(["ousast", "pre-push", "--help"])
|
||||
assert help_result.returncode == 0, help_result.stderr
|
||||
(out / "help.txt").write_bytes(help_result.stdout)
|
||||
current = _provenance(out, ScanBudget(max_model_calls=0, max_regions=500, order_by_evidence=True))
|
||||
registry = load_registry(current=current)
|
||||
assert _DEFAULT.is_file() and _DEFAULT.stat().st_size > 0
|
||||
assert registry.status == "NO-GO" and not registry.capabilities, registry
|
||||
results = []
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
for prior_exit in (0, 7):
|
||||
root = tmp / str(prior_exit)
|
||||
root.mkdir()
|
||||
|
||||
def git(*args, root=root):
|
||||
result = run(["git", "-C", str(root), *args])
|
||||
assert result.returncode == 0, result.stderr
|
||||
return result.stdout.decode().strip()
|
||||
|
||||
git("init")
|
||||
git("config", "user.name", "Packaged smoke")
|
||||
git("config", "user.email", "[email protected]")
|
||||
(root / "api.js").write_text("function endpoint(req) { return eval(req.body.value); }\n")
|
||||
assert len((root / "api.js").read_bytes()) > 0
|
||||
git("add", "api.js")
|
||||
git("commit", "-m", "Authored interface smoke, not a security control")
|
||||
head = git("rev-parse", "HEAD")
|
||||
remote = root / "remote.git"
|
||||
git("init", "--bare", str(remote))
|
||||
git("remote", "add", "origin", str(remote))
|
||||
git("config", "core.hooksPath", ".custom-hooks")
|
||||
installer = run([str(args.ops / "install-pre-push"), str(root)])
|
||||
assert installer.returncode == 0, installer.stderr
|
||||
hook = root / ".custom-hooks/pre-push"
|
||||
before_hook = hook.read_bytes()
|
||||
refused = run([str(args.ops / "install-pre-push"), str(root)])
|
||||
assert refused.returncode != 0 and hook.read_bytes() == before_hook
|
||||
prior = hook.with_name("pre-push.before-ousast")
|
||||
prior.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > prior-args\ncat > prior-stdin\nexit ' + str(prior_exit) + "\n")
|
||||
prior.chmod(0o755)
|
||||
before = ((root / ".git/index").read_bytes(), (root / "api.js").read_bytes(), head)
|
||||
artifacts = out / str(prior_exit)
|
||||
env = {**os.environ, "OUSAST_ARTIFACT_DIR": str(artifacts), "OUSAST_PUSH_DEADLINE": "0.1"}
|
||||
result = run(["git", "-C", str(root), "push", "origin", "HEAD:refs/heads/smoke"], env=env)
|
||||
assert (result.returncode == 0) == (prior_exit == 0), result.stderr
|
||||
assert (root / "prior-stdin").read_text() == f"HEAD {head} refs/heads/smoke {'0' * 40}\n"
|
||||
assert (root / "prior-args").read_text().splitlines() == ["origin", str(remote)]
|
||||
assert before == ((root / ".git/index").read_bytes(), (root / "api.js").read_bytes(), git("rev-parse", "HEAD"))
|
||||
assert git("config", "core.hooksPath") == ".custom-hooks"
|
||||
saved = list(artifacts.glob("*.json"))
|
||||
assert len(saved) == 1
|
||||
artifact = json.loads(saved[0].read_bytes())
|
||||
assert not artifact["admission"]["defects"]
|
||||
assert artifact["result"]["coverage_status"] != "complete_within_scope"
|
||||
original = prior.read_bytes()
|
||||
hook.unlink()
|
||||
prior.rename(hook)
|
||||
assert hook.read_bytes() == original
|
||||
after = run(["git", "-C", str(root), "push", "origin", "HEAD:refs/heads/restored"], env=env)
|
||||
assert (after.returncode == 0) == (prior_exit == 0)
|
||||
results.append(
|
||||
dict(
|
||||
prior_exit=prior_exit,
|
||||
push_exit=result.returncode,
|
||||
removal_push_exit=after.returncode,
|
||||
exact_stdin=True,
|
||||
exact_remote_args=True,
|
||||
live_state_preserved=True,
|
||||
overwrite_refused=True,
|
||||
coverage=artifact["result"]["coverage_status"],
|
||||
source_bytes=len(before[1]),
|
||||
terminal=(result.stdout + result.stderr).decode(),
|
||||
)
|
||||
)
|
||||
receipt = dict(
|
||||
uid=os.getuid(),
|
||||
provenance=current,
|
||||
registry_status=registry.status,
|
||||
enabled=0,
|
||||
no_model_configuration=True,
|
||||
network="disabled by docker invocation",
|
||||
cases=results,
|
||||
)
|
||||
(out / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
|
||||
print(json.dumps(receipt))
|
||||
Reference in New Issue
Block a user