From 8acada0d9831159e1083aed6867c05fa75968a8b Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:45:54 +0200 Subject: [PATCH] feat(pre-push-safety-net): pin labeled evaluation inputs --- .kiro/specs/pre-push-safety-net/evaluation.md | 6 +- .kiro/specs/pre-push-safety-net/tasks.md | 3 +- .../measurements/2026-09-08-module-audit.json | 18 +- benchmarks/push/README.md | 46 ++ benchmarks/push/ghost-reserved.toml | 8 + benchmarks/push/input-validation.json | 414 ++++++++++++++++++ benchmarks/push/inputs.json | 388 ++++++++++++++++ benchmarks/repos/nodegoat.toml | 17 + src/openultrasast/model/audit.py | 1 + src/openultrasast/push_inputs.py | 158 +++++++ tests/test_push_inputs.py | 141 ++++++ 11 files changed, 1194 insertions(+), 6 deletions(-) create mode 100644 benchmarks/push/README.md create mode 100644 benchmarks/push/ghost-reserved.toml create mode 100644 benchmarks/push/input-validation.json create mode 100644 benchmarks/push/inputs.json create mode 100644 benchmarks/repos/nodegoat.toml create mode 100644 src/openultrasast/push_inputs.py create mode 100644 tests/test_push_inputs.py diff --git a/.kiro/specs/pre-push-safety-net/evaluation.md b/.kiro/specs/pre-push-safety-net/evaluation.md index cd279ff..8a3f125 100644 --- a/.kiro/specs/pre-push-safety-net/evaluation.md +++ b/.kiro/specs/pre-push-safety-net/evaluation.md @@ -1,7 +1,7 @@ # Evaluation: core optimization and cross-language transfer Status: evaluation contract accepted with requirements/design for task generation, 2026-09-12. Implements requirements 6.1–6.5 and 8.1–8.4. -No new repository has been pinned, scanned or admitted by this document. +Task 1.2 now pins development and reserved inputs in `benchmarks/push/inputs.json`; see its README for provenance and missing prerequisites. No new scanner result or capability admission is claimed. ## Workload roles @@ -80,3 +80,7 @@ graph exclusion policy, deadline and admission contract. Add at least one benign Produce one joint scorecard before any further core optimization. A missing Node frontend, broken middleware mapping or wrong sanitizer model is an explicit finding of this experiment, not a reason to remove Node from the evaluation or silently fall back to a PHP-only release claim. + +## Task 1.2 input preparation + +The versioned push manifest records upstream PMPro vulnerable/fixed revisions, NodeGoat with an explicitly authored eval-removal repair, and comment-only benign changes that retain vulnerable backlog. Ghost is reserved independently for production Node module/async API evaluation; its reviewed supported-family pair population remains missing. The validator verifies immutable Git blob identities and source/license byte counts offline, retains every unresolved case, and keeps all capabilities experimental. Independent PHP and Python admission populations remain prerequisites; VAmPI regressions and the libpng unsupported-property envelope retain their separate roles. See `benchmarks/push/README.md` for the reproducible command and provenance boundaries. diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index d533740..beec7ad 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -29,7 +29,7 @@ started with group 1 (tasks 1.1–1.4); each task remains pending until independ - _Boundary: Cache and Engine Boundary, packaging integration_ - _Requirements: 7.1, 7.3, 8.2_ - _Verified 2026-09-12:_ independent review APPROVED; 888 tests passed, 9 skipped. Actual local image PHP/JavaScript graph census and source witnesses passed, unreadable PHP explicitly refused. Parent verified current provenance hashes, image identity, non-root user and CLI/frontend liveness. Evidence: `benchmarks/measurements/2026-09-12-engine-runtime-smoke.json`. Runtime readiness only; no hook latency/detection claim. Existing mypy errors in `model/scan.py` at 299/302/310 remain unchanged. -- [ ] 1.2 Prepare labeled development and reserved evaluation changes +- [x] 1.2 Prepare labeled development and reserved evaluation changes - Pin a readable PMPro change/fix and a supported Node/Express change/fix, plus benign controls; verify license and source bytes before accepting recipes. - Use NodeGoat only if its selected mechanism fits a current arbiter; label authored repairs as authored. Reserve an independently selected Node API repository with modern module/async context and reviewable history before tuning on transfer results. - Keep Python regression and libpng envelope recipes, explicit supported/unsupported populations, ref identities and development/holdout roles in machine-readable evaluation inputs. @@ -37,6 +37,7 @@ started with group 1 (tasks 1.1–1.4); each task remains pending until independ - Done when recipe validation resolves immutable inputs and labels or names an explicit missing prerequisite; no favorable scanner result is required to select a recipe. - _Boundary: Evaluation harness_ - _Requirements: 6.1, 6.3, 8.3, 8.4_ + - _Verified 2026-09-12:_ independent review APPROVED; 902 tests passed, 9 skipped, then 34 focused tests passed after a diagnostic-only correction. Fresh offline validation opened 506,502 bytes and retained all 11 cases/9 snapshots: 7 cases input-ready, 4 with named missing prerequisites. See `benchmarks/push/inputs.json` and `input-validation.json`. Ghost checkout/reviewed labels and independent per-capability PHP/Python populations remain missing; every capability stays experimental. No detector run or admission claim. - [ ] 1.3 Make the baseline and push scorer fail honestly - Retain unmatched targets and unanswered queries in the declared population; require a queried witness before labeling a target transitively detected. - Emit actual vulnerable/fixed outcomes beside rank, selected mode, engine/facts/scope and elapsed stage costs. Freeze a versioned workload/profile before measuring. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 0b05155..95e8285 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,9 +1,9 @@ { "counts": { "load_bearing": 92, - "standalone_capability": 7 + "standalone_capability": 8 }, - "modules": 99, + "modules": 100, "orphaned": [], "rows": [ { @@ -604,6 +604,12 @@ "module": "provider.openrouter", "reason": "reachable from gate, map_gate, pair_gate or the CLI" }, + { + "classification": "standalone_capability", + "importers": [], + "module": "push_inputs", + "reason": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs" + }, { "classification": "load_bearing", "importers": [ @@ -689,7 +695,8 @@ { "classification": "load_bearing", "importers": [ - "cli" + "cli", + "push_inputs" ], "module": "repos", "reason": "reachable from gate, map_gate, pair_gate or the CLI" @@ -1068,8 +1075,11 @@ "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs", "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", "slot_contract": "the slot/contract protocol stage_processors implements", "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" - } + }, + "update_note": "Current tree inventory refreshed for pre-push-safety-net task 1.2 standalone input validator; filename retains the original audit date. No detection or ranking measurement.", + "updated_date": "2026-09-12" } diff --git a/benchmarks/push/README.md b/benchmarks/push/README.md new file mode 100644 index 0000000..d9a2f6e --- /dev/null +++ b/benchmarks/push/README.md @@ -0,0 +1,46 @@ +# Reproducible push evaluation inputs + +`inputs.json` schema 1 declares 11 cases and 9 snapshots before any new scanner run. +Run `.venv/bin/python -m openultrasast.push_inputs benchmarks/push/inputs.json`. +It is offline by default, even if the ordinary repository network environment variable is set. +`--fetch` explicitly permits the existing repository resolver to fetch missing pinned commits. +No project scripts, npm installation, or scanner are executed. Nonzero exit means at least one +case has invalid input or an explicit missing prerequisite; the complete population stays in JSON. + +Each snapshot references an existing repository recipe (with an optional full commit override), +plus a census of readable witness/context/license blobs. The commit identifies the whole tree; +the file list is provenance evidence, **not an analysis scope or graph allowlist**. Future replay +must materialize the pinned tree and apply declared authored edits before the ranker chooses scope. +Bytes come from `git cat-file` at the full SHA rather than the mutable cached working tree. +SHA-256 and byte length are checked for every listed blob. An authored replacement must match +exactly once, declare provenance, and verify both the original and resulting identity. + +PMPro uses upstream 2.9.7 and 2.9.8. Its SQL query changes from interpolation to `$wpdb->prepare`. +The introducing replay reverses the reviewed pair; it does not claim that 2.9.7 historically +introduced the CVE. NodeGoat's contribution handler evaluates three request fields before numeric +validation. Its authored repair replaces these eval calls with `Number`, eliminating JavaScript +execution; it is not represented as an upstream security fix or a comprehensive numeric-validation +repair. Its commented tutorial repair remains unchanged. Existing injection facts include JavaScript +`eval`; actual frontend, ranker and arbiter behavior remains to be measured. + +Both benign variants change only a comment on the **vulnerable** base. They mean no new defect, +not repository-clean, and deliberately retain backlog to test whether the hook becomes a nag. +The PMPro source pair is GPL-2.0; NodeGoat is Apache-2.0. Only hashes and small authored edit +instructions are stored here; repository source stays in the external opt-in cache. + +Ghost is reserved independently for its production Node API, CommonJS module boundaries and async +endpoint methods, with public reviewable history at +https://github.com/TryGhost/Ghost/security/advisories. The pin declares Ghost 6.64.0-rc.0 and +Express 4.22.2. It does not establish ESM or Express 5 coverage. Selection inspected only the +manifest, license and endpoint shape, not scanner results. Their source bytes were read at the +pinned raw GitHub URLs and hashed; the offline validator still requires Git object availability. +Its supported-family positive/fixed/benign population must be independently reviewed and frozen +before scanning. If inspected results inform tuning, retire its holdout status. No reserved +security case is silently inferred from the existence of an advisory. + +All capabilities remain experimental: NodeGoat is inspected transfer/development data and cannot +qualify production Node support; Node evidence cannot qualify PHP or Python. Independent PHP and +Python populations are explicitly missing in `capability_prerequisites`. VAmPI retains all three +known targets (including SQLI), with `vulnerable=1`; these snapshot regressions are not paired pushes. +Libpng is an envelope/unsupported-arithmetic control, not a detection success or a supported miss. +The validator makes no finding, precision, recall, silence, latency or admission claim. diff --git a/benchmarks/push/ghost-reserved.toml b/benchmarks/push/ghost-reserved.toml new file mode 100644 index 0000000..abcd75f --- /dev/null +++ b/benchmarks/push/ghost-reserved.toml @@ -0,0 +1,8 @@ +name = "ghost-reserved" +url = "https://github.com/TryGhost/Ghost.git" +commit = "07671c072c5a5e0538cbe44d9d4ca66111751535" +language = "javascript" +license = "MIT" +license_file = "LICENSE" +measures = ["envelope"] +note = "Reserved independently before scanner evaluation: production Node API, CommonJS modules and async endpoints; public security-fix history at https://github.com/TryGhost/Ghost/security/advisories. Only source/manifest/license provenance inspected. No detection case or capability admission is asserted. Reviewed positive/fixed/benign labels remain an explicit prerequisite." diff --git a/benchmarks/push/input-validation.json b/benchmarks/push/input-validation.json new file mode 100644 index 0000000..5847a99 --- /dev/null +++ b/benchmarks/push/input-validation.json @@ -0,0 +1,414 @@ +{ + "schema_version": 1, + "manifest_sha256": "eec13cb19fe5e2c2402e61ebf4fd17fa809482f321e7a8e615215a12e1bcf845", + "population": 11, + "capability_prerequisites": { + "php/wordpress/injection": [ + "Independent untouched PHP/WordPress vulnerable/fixed/benign population missing." + ], + "python/flask/injection": [ + "Independent untouched Python/Flask paired changes missing." + ], + "python/flask/access_control": [ + "Independent untouched Python/Flask paired changes missing." + ], + "javascript/express/injection": [ + "Ghost supported-family pair review missing; NodeGoat is inspected teaching data." + ], + "c/libpng/memory_safety": [ + "Unsupported abstract domain; envelope cannot earn detection admission." + ] + }, + "snapshots": [ + { + "id": "pmpro-vulnerable", + "status": "ready", + "bytes_read": 93181, + "files": [ + { + "path": "license.txt", + "size": 1039, + "sha256": "98424ae7dd8610cd5e3a29dd2775347552444e4793653823bc74c010d39265ff" + }, + { + "path": "classes/class.memberorder.php", + "size": 55918, + "sha256": "3d32b8414eb9a6bfe8962aafa8ab2caf826d2816811caf51d299db3398f54c26" + }, + { + "path": "includes/rest-api.php", + "size": 36224, + "sha256": "f52c146967d1d154b65dc5fb289ee831a7a0d31d7a9bba4876f40eee903624b0" + } + ], + "commit": "1a900be3f09f243876150fc3c04a97bcb24ef84b", + "repository": "https://github.com/strangerstudios/paid-memberships-pro.git", + "license": "GPL-2.0" + }, + { + "id": "pmpro-fixed", + "status": "ready", + "bytes_read": 93465, + "files": [ + { + "path": "license.txt", + "size": 1039, + "sha256": "98424ae7dd8610cd5e3a29dd2775347552444e4793653823bc74c010d39265ff" + }, + { + "path": "classes/class.memberorder.php", + "size": 56202, + "sha256": "6877bdc6f9322fe990be8aedfd1778718fc3e06b67c96eff47be16824eb9b312" + }, + { + "path": "includes/rest-api.php", + "size": 36224, + "sha256": "f52c146967d1d154b65dc5fb289ee831a7a0d31d7a9bba4876f40eee903624b0" + } + ], + "commit": "2f65530f9442b4d7ccfb56821836b056217358c1", + "repository": "https://github.com/strangerstudios/paid-memberships-pro.git", + "license": "GPL-2.0" + }, + { + "id": "pmpro-benign", + "status": "ready", + "bytes_read": 93181, + "files": [ + { + "path": "license.txt", + "size": 1039, + "sha256": "98424ae7dd8610cd5e3a29dd2775347552444e4793653823bc74c010d39265ff" + }, + { + "path": "classes/class.memberorder.php", + "size": 55962, + "sha256": "457cfa61eddb5d339502143ab59fd93db93d577a1557f6552081c6a6fc2ebf8b" + }, + { + "path": "includes/rest-api.php", + "size": 36224, + "sha256": "f52c146967d1d154b65dc5fb289ee831a7a0d31d7a9bba4876f40eee903624b0" + } + ], + "commit": "1a900be3f09f243876150fc3c04a97bcb24ef84b", + "repository": "https://github.com/strangerstudios/paid-memberships-pro.git", + "license": "GPL-2.0" + }, + { + "id": "nodegoat-vulnerable", + "status": "ready", + "bytes_read": 20677, + "files": [ + { + "path": "LICENSE", + "size": 10273, + "sha256": "73ba74dfaa520b49a401b5d21459a8523a146f3b7518a833eea5efa85130bf68" + }, + { + "path": "package.json", + "size": 1979, + "sha256": "138377cf0f19b955d6ba043e49c24fb5eec4041a789fea5134e1fecda620c829" + }, + { + "path": "app/routes/index.js", + "size": 3149, + "sha256": "3ab970b2888bc1501515db24a83cc7851ac9c09d00948c652300401515185eb5" + }, + { + "path": "app/routes/contributions.js", + "size": 2485, + "sha256": "196bdeaa22da4cfe8a851b1b932710b2fcedd21f14ed75d70b739ba759a01003" + }, + { + "path": "app/data/contributions-dao.js", + "size": 2791, + "sha256": "363065e8b3c32f0997ce2dcb17341eb3e40b7f43b00b72c576dfebf73f551c85" + } + ], + "commit": "c5cb68a7084e4ae7dcc60e6a98768720a81841e8", + "repository": "https://github.com/OWASP/NodeGoat.git", + "license": "Apache-2.0" + }, + { + "id": "nodegoat-fixed", + "status": "ready", + "bytes_read": 20677, + "files": [ + { + "path": "LICENSE", + "size": 10273, + "sha256": "73ba74dfaa520b49a401b5d21459a8523a146f3b7518a833eea5efa85130bf68" + }, + { + "path": "package.json", + "size": 1979, + "sha256": "138377cf0f19b955d6ba043e49c24fb5eec4041a789fea5134e1fecda620c829" + }, + { + "path": "app/routes/index.js", + "size": 3149, + "sha256": "3ab970b2888bc1501515db24a83cc7851ac9c09d00948c652300401515185eb5" + }, + { + "path": "app/routes/contributions.js", + "size": 2491, + "sha256": "9dc0a0bc147adc4c76c27faae75f316dd537ddaf4fb35fa693002450bd4b4e92" + }, + { + "path": "app/data/contributions-dao.js", + "size": 2791, + "sha256": "363065e8b3c32f0997ce2dcb17341eb3e40b7f43b00b72c576dfebf73f551c85" + } + ], + "commit": "c5cb68a7084e4ae7dcc60e6a98768720a81841e8", + "repository": "https://github.com/OWASP/NodeGoat.git", + "license": "Apache-2.0" + }, + { + "id": "nodegoat-benign", + "status": "ready", + "bytes_read": 20677, + "files": [ + { + "path": "LICENSE", + "size": 10273, + "sha256": "73ba74dfaa520b49a401b5d21459a8523a146f3b7518a833eea5efa85130bf68" + }, + { + "path": "package.json", + "size": 1979, + "sha256": "138377cf0f19b955d6ba043e49c24fb5eec4041a789fea5134e1fecda620c829" + }, + { + "path": "app/routes/index.js", + "size": 3149, + "sha256": "3ab970b2888bc1501515db24a83cc7851ac9c09d00948c652300401515185eb5" + }, + { + "path": "app/routes/contributions.js", + "size": 2498, + "sha256": "66b97183be80e55fa3d690d2fc7103aabddb4e10ef9db439c22bce6c7651bcc4" + }, + { + "path": "app/data/contributions-dao.js", + "size": 2791, + "sha256": "363065e8b3c32f0997ce2dcb17341eb3e40b7f43b00b72c576dfebf73f551c85" + } + ], + "commit": "c5cb68a7084e4ae7dcc60e6a98768720a81841e8", + "repository": "https://github.com/OWASP/NodeGoat.git", + "license": "Apache-2.0" + }, + { + "id": "vampi-regression", + "status": "ready", + "bytes_read": 17541, + "files": [ + { + "path": "LICENSE", + "size": 1063, + "sha256": "e3e05b9d17e397de650aa7a9a9784cf076ba8bf566efebe88a0e24ecb2d2b92a" + }, + { + "path": "models/user_model.py", + "size": 3642, + "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b" + }, + { + "path": "api_views/users.py", + "size": 9257, + "sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c" + }, + { + "path": "api_views/books.py", + "size": 3043, + "sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8" + }, + { + "path": "app.py", + "size": 536, + "sha256": "ad26b12cfbdaab311bea86942f5ebd368dca331a0f4cdba803393739baadf42e" + } + ], + "commit": "f16052dce83f05847133ec98f01c5193a41de7d8", + "repository": "https://github.com/erev0s/VAmPI.git", + "license": "MIT" + }, + { + "id": "libpng-envelope", + "status": "ready", + "bytes_read": 147103, + "files": [ + { + "path": "LICENSE", + "size": 5345, + "sha256": "16d9daaafbf63a31a5bdc91d4600972548fef5aaa1244202393288dbd079c49a" + }, + { + "path": "pngread.c", + "size": 141758, + "sha256": "c12022962f04abe1f7156753315bf68091398711025450803d0506dcefda3552" + } + ], + "commit": "cf53e54ccf631d8de68a30ebb3b8eeed1d6f1a08", + "repository": "https://github.com/glennrp/libpng.git", + "license": "libpng-2.0" + }, + { + "id": "ghost-reserved", + "status": "missing_prerequisite", + "bytes_read": 0, + "files": [], + "problem": "ghost-reserved: pinned checkout 07671c072c5a5e0538cbe44d9d4ca66111751535 unavailable; rerun this validator with --fetch to fetch it" + } + ], + "cases": [ + { + "id": "pmpro-introduce", + "base": "pmpro-fixed", + "tip": "pmpro-vulnerable", + "label": "vulnerable", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "pmpro-repair", + "base": "pmpro-vulnerable", + "tip": "pmpro-fixed", + "label": "fixed", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "PMPro upstream2.9.7 to2.9.8 parameter binding fix for CVE-2023-23488.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "pmpro-benign", + "base": "pmpro-vulnerable", + "tip": "pmpro-benign", + "label": "benign", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "nodegoat-introduce", + "base": "nodegoat-fixed", + "tip": "nodegoat-vulnerable", + "label": "vulnerable", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "nodegoat-repair", + "base": "nodegoat-vulnerable", + "tip": "nodegoat-fixed", + "label": "fixed", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored replacement of three eval calls with Number conversion; source tutorial also documents alternate numeric parsing.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "nodegoat-benign", + "base": "nodegoat-vulnerable", + "tip": "nodegoat-benign", + "label": "benign", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "VAmPI-SQLI", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/injection", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ], + "status": "missing_prerequisite", + "admission": "experimental", + "problem": "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + }, + { + "id": "VAmPI-BOLA", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/access_control", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ], + "status": "missing_prerequisite", + "admission": "experimental", + "problem": "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + }, + { + "id": "VAmPI-BOLA-BOOKS", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/access_control", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ], + "status": "missing_prerequisite", + "admission": "experimental", + "problem": "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + }, + { + "id": "libpng-CVE-2025-66293", + "base": "libpng-envelope", + "tip": "libpng-envelope", + "label": "unsupported", + "role": "envelope", + "capability": "c/libpng/memory_safety", + "provenance": "Arithmetic/bounds is outside the current abstract domain; measures graph/cost envelope only.", + "prerequisites": [], + "status": "ready", + "admission": "experimental" + }, + { + "id": "ghost-reserved-population", + "base": "ghost-reserved", + "tip": "ghost-reserved", + "label": "unreviewed", + "role": "reserved_holdout", + "capability": "javascript/express/unreviewed", + "provenance": "Reserved from production API/module/async shape and public security-fix history before scanner results. Manifest6.64.0-rc.0, Express4.22.2. Not an ESM or Express5 coverage claim.", + "prerequisites": [ + "Review and freeze supported positive/fixed/benign changes from independent history; select family before scanning.", + "Freeze core and existing facts after development; preserve reservation and record any contamination." + ], + "status": "missing_prerequisite", + "admission": "experimental", + "problem": "ghost-reserved: pinned checkout 07671c072c5a5e0538cbe44d9d4ca66111751535 unavailable; rerun this validator with --fetch to fetch it; ghost-reserved: pinned checkout 07671c072c5a5e0538cbe44d9d4ca66111751535 unavailable; rerun this validator with --fetch to fetch it; Review and freeze supported positive/fixed/benign changes from independent history; select family before scanning.; Freeze core and existing facts after development; preserve reservation and record any contamination." + } + ], + "note": "Input validation only; every capability remains experimental pending independent evaluation." +} diff --git a/benchmarks/push/inputs.json b/benchmarks/push/inputs.json new file mode 100644 index 0000000..cefdb5c --- /dev/null +++ b/benchmarks/push/inputs.json @@ -0,0 +1,388 @@ +{ + "schema_version": 1, + "selection_date": "2026-09-12", + "selection_basis": "PMPro optimization plus NodeGoat documented eval injection; Ghost independent production Node module/async reserve. No scanner results used for new selection.", + "capability_prerequisites": { + "php/wordpress/injection": [ + "Independent untouched PHP/WordPress vulnerable/fixed/benign population missing." + ], + "python/flask/injection": [ + "Independent untouched Python/Flask paired changes missing." + ], + "python/flask/access_control": [ + "Independent untouched Python/Flask paired changes missing." + ], + "javascript/express/injection": [ + "Ghost supported-family pair review missing; NodeGoat is inspected teaching data." + ], + "c/libpng/memory_safety": [ + "Unsupported abstract domain; envelope cannot earn detection admission." + ] + }, + "snapshots": [ + { + "id": "pmpro-vulnerable", + "recipe": "../repos/pmpro.toml", + "files": [ + { + "path": "license.txt", + "size": 1039, + "sha256": "98424ae7dd8610cd5e3a29dd2775347552444e4793653823bc74c010d39265ff" + }, + { + "path": "classes/class.memberorder.php", + "size": 55918, + "sha256": "3d32b8414eb9a6bfe8962aafa8ab2caf826d2816811caf51d299db3398f54c26" + }, + { + "path": "includes/rest-api.php", + "size": 36224, + "sha256": "f52c146967d1d154b65dc5fb289ee831a7a0d31d7a9bba4876f40eee903624b0" + } + ] + }, + { + "id": "pmpro-fixed", + "recipe": "../repos/pmpro.toml", + "files": [ + { + "path": "license.txt", + "size": 1039, + "sha256": "98424ae7dd8610cd5e3a29dd2775347552444e4793653823bc74c010d39265ff" + }, + { + "path": "classes/class.memberorder.php", + "size": 56202, + "sha256": "6877bdc6f9322fe990be8aedfd1778718fc3e06b67c96eff47be16824eb9b312" + }, + { + "path": "includes/rest-api.php", + "size": 36224, + "sha256": "f52c146967d1d154b65dc5fb289ee831a7a0d31d7a9bba4876f40eee903624b0" + } + ], + "commit": "2f65530f9442b4d7ccfb56821836b056217358c1" + }, + { + "id": "pmpro-benign", + "recipe": "../repos/pmpro.toml", + "files": [ + { + "path": "license.txt", + "size": 1039, + "sha256": "98424ae7dd8610cd5e3a29dd2775347552444e4793653823bc74c010d39265ff" + }, + { + "path": "classes/class.memberorder.php", + "original_sha256": "3d32b8414eb9a6bfe8962aafa8ab2caf826d2816811caf51d299db3398f54c26", + "edit": { + "before": "function getMemberOrderByCode($code)", + "after": "/* Load an order by its external code. */\n\t\tfunction getMemberOrderByCode($code)", + "provenance": "authored comment-only change; existing SQL injection remains" + }, + "size": 55962, + "sha256": "457cfa61eddb5d339502143ab59fd93db93d577a1557f6552081c6a6fc2ebf8b" + }, + { + "path": "includes/rest-api.php", + "size": 36224, + "sha256": "f52c146967d1d154b65dc5fb289ee831a7a0d31d7a9bba4876f40eee903624b0" + } + ] + }, + { + "id": "nodegoat-vulnerable", + "recipe": "../repos/nodegoat.toml", + "files": [ + { + "path": "LICENSE", + "size": 10273, + "sha256": "73ba74dfaa520b49a401b5d21459a8523a146f3b7518a833eea5efa85130bf68" + }, + { + "path": "package.json", + "size": 1979, + "sha256": "138377cf0f19b955d6ba043e49c24fb5eec4041a789fea5134e1fecda620c829" + }, + { + "path": "app/routes/index.js", + "size": 3149, + "sha256": "3ab970b2888bc1501515db24a83cc7851ac9c09d00948c652300401515185eb5" + }, + { + "path": "app/routes/contributions.js", + "size": 2485, + "sha256": "196bdeaa22da4cfe8a851b1b932710b2fcedd21f14ed75d70b739ba759a01003" + }, + { + "path": "app/data/contributions-dao.js", + "size": 2791, + "sha256": "363065e8b3c32f0997ce2dcb17341eb3e40b7f43b00b72c576dfebf73f551c85" + } + ] + }, + { + "id": "nodegoat-fixed", + "recipe": "../repos/nodegoat.toml", + "files": [ + { + "path": "LICENSE", + "size": 10273, + "sha256": "73ba74dfaa520b49a401b5d21459a8523a146f3b7518a833eea5efa85130bf68" + }, + { + "path": "package.json", + "size": 1979, + "sha256": "138377cf0f19b955d6ba043e49c24fb5eec4041a789fea5134e1fecda620c829" + }, + { + "path": "app/routes/index.js", + "size": 3149, + "sha256": "3ab970b2888bc1501515db24a83cc7851ac9c09d00948c652300401515185eb5" + }, + { + "path": "app/routes/contributions.js", + "original_sha256": "196bdeaa22da4cfe8a851b1b932710b2fcedd21f14ed75d70b739ba759a01003", + "edit": { + "before": "const preTax = eval(req.body.preTax);\n const afterTax = eval(req.body.afterTax);\n const roth = eval(req.body.roth);", + "after": "const preTax = Number(req.body.preTax);\n const afterTax = Number(req.body.afterTax);\n const roth = Number(req.body.roth);", + "provenance": "authored numeric conversion repair removes evaluation of JavaScript; not an upstream CVE fix" + }, + "size": 2491, + "sha256": "9dc0a0bc147adc4c76c27faae75f316dd537ddaf4fb35fa693002450bd4b4e92" + }, + { + "path": "app/data/contributions-dao.js", + "size": 2791, + "sha256": "363065e8b3c32f0997ce2dcb17341eb3e40b7f43b00b72c576dfebf73f551c85" + } + ] + }, + { + "id": "nodegoat-benign", + "recipe": "../repos/nodegoat.toml", + "files": [ + { + "path": "LICENSE", + "size": 10273, + "sha256": "73ba74dfaa520b49a401b5d21459a8523a146f3b7518a833eea5efa85130bf68" + }, + { + "path": "package.json", + "size": 1979, + "sha256": "138377cf0f19b955d6ba043e49c24fb5eec4041a789fea5134e1fecda620c829" + }, + { + "path": "app/routes/index.js", + "size": 3149, + "sha256": "3ab970b2888bc1501515db24a83cc7851ac9c09d00948c652300401515185eb5" + }, + { + "path": "app/routes/contributions.js", + "original_sha256": "196bdeaa22da4cfe8a851b1b932710b2fcedd21f14ed75d70b739ba759a01003", + "edit": { + "before": "//validate contributions", + "after": "// Validate contribution percentages.", + "provenance": "authored comment-only change; existing eval injection remains" + }, + "size": 2498, + "sha256": "66b97183be80e55fa3d690d2fc7103aabddb4e10ef9db439c22bce6c7651bcc4" + }, + { + "path": "app/data/contributions-dao.js", + "size": 2791, + "sha256": "363065e8b3c32f0997ce2dcb17341eb3e40b7f43b00b72c576dfebf73f551c85" + } + ] + }, + { + "id": "vampi-regression", + "recipe": "../repos/vampi.toml", + "files": [ + { + "path": "LICENSE", + "size": 1063, + "sha256": "e3e05b9d17e397de650aa7a9a9784cf076ba8bf566efebe88a0e24ecb2d2b92a" + }, + { + "path": "models/user_model.py", + "size": 3642, + "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b" + }, + { + "path": "api_views/users.py", + "size": 9257, + "sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c" + }, + { + "path": "api_views/books.py", + "size": 3043, + "sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8" + }, + { + "path": "app.py", + "size": 536, + "sha256": "ad26b12cfbdaab311bea86942f5ebd368dca331a0f4cdba803393739baadf42e" + } + ] + }, + { + "id": "libpng-envelope", + "recipe": "../repos/libpng.toml", + "files": [ + { + "path": "LICENSE", + "size": 5345, + "sha256": "16d9daaafbf63a31a5bdc91d4600972548fef5aaa1244202393288dbd079c49a" + }, + { + "path": "pngread.c", + "size": 141758, + "sha256": "c12022962f04abe1f7156753315bf68091398711025450803d0506dcefda3552" + } + ] + }, + { + "id": "ghost-reserved", + "recipe": "ghost-reserved.toml", + "files": [ + { + "path": "LICENSE", + "size": 1065, + "sha256": "56df280a26071c1e84578abcb1e7dce02502d27816bce1184617e93ee5b086bf" + }, + { + "path": "ghost/core/package.json", + "size": 12646, + "sha256": "f63492ad63f64ecf0c3648f57e196b904af1d79855d296ec3db659aae8cd5701" + }, + { + "path": "ghost/core/core/server/api/endpoints/posts.js", + "size": 8008, + "sha256": "f51acc472f2fb217acd99a3bd378f6b6117d19f4c50233c58cf22538f3ea864f" + } + ] + } + ], + "cases": [ + { + "id": "pmpro-introduce", + "base": "pmpro-fixed", + "tip": "pmpro-vulnerable", + "label": "vulnerable", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.", + "prerequisites": [] + }, + { + "id": "pmpro-repair", + "base": "pmpro-vulnerable", + "tip": "pmpro-fixed", + "label": "fixed", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "PMPro upstream2.9.7 to2.9.8 parameter binding fix for CVE-2023-23488.", + "prerequisites": [] + }, + { + "id": "pmpro-benign", + "base": "pmpro-vulnerable", + "tip": "pmpro-benign", + "label": "benign", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.", + "prerequisites": [] + }, + { + "id": "nodegoat-introduce", + "base": "nodegoat-fixed", + "tip": "nodegoat-vulnerable", + "label": "vulnerable", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.", + "prerequisites": [] + }, + { + "id": "nodegoat-repair", + "base": "nodegoat-vulnerable", + "tip": "nodegoat-fixed", + "label": "fixed", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored replacement of three eval calls with Number conversion; source tutorial also documents alternate numeric parsing.", + "prerequisites": [] + }, + { + "id": "nodegoat-benign", + "base": "nodegoat-vulnerable", + "tip": "nodegoat-benign", + "label": "benign", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.", + "prerequisites": [] + }, + { + "id": "VAmPI-SQLI", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/injection", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ] + }, + { + "id": "VAmPI-BOLA", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/access_control", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ] + }, + { + "id": "VAmPI-BOLA-BOOKS", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/access_control", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ] + }, + { + "id": "libpng-CVE-2025-66293", + "base": "libpng-envelope", + "tip": "libpng-envelope", + "label": "unsupported", + "role": "envelope", + "capability": "c/libpng/memory_safety", + "provenance": "Arithmetic/bounds is outside the current abstract domain; measures graph/cost envelope only.", + "prerequisites": [] + }, + { + "id": "ghost-reserved-population", + "base": "ghost-reserved", + "tip": "ghost-reserved", + "label": "unreviewed", + "role": "reserved_holdout", + "capability": "javascript/express/unreviewed", + "provenance": "Reserved from production API/module/async shape and public security-fix history before scanner results. Manifest6.64.0-rc.0, Express4.22.2. Not an ESM or Express5 coverage claim.", + "prerequisites": [ + "Review and freeze supported positive/fixed/benign changes from independent history; select family before scanning.", + "Freeze core and existing facts after development; preserve reservation and record any contamination." + ] + } + ] +} diff --git a/benchmarks/repos/nodegoat.toml b/benchmarks/repos/nodegoat.toml new file mode 100644 index 0000000..429d826 --- /dev/null +++ b/benchmarks/repos/nodegoat.toml @@ -0,0 +1,17 @@ +name = "nodegoat" +url = "https://github.com/OWASP/NodeGoat.git" +commit = "c5cb68a7084e4ae7dcc60e6a98768720a81841e8" +language = "javascript" +license = "Apache-2.0" +license_file = "LICENSE" +measures = ["detection"] +note = "Express teaching/development transfer input, selected from documented eval injection without scanner results. Not a production holdout. Authored repair and benign variants are in benchmarks/push/inputs.json." +[[known]] +id = "NodeGoat-contribution-eval" +family = "injection" +weakness = "Request contribution fields are evaluated as JavaScript before numeric validation." +file = "app/routes/contributions.js" +function = "handleContributionsUpdate" +line = 32 +reference = "The pinned source documents SSJS injection and an alternate numeric parsing repair immediately below the eval calls." +in_scope = true diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index 70505b3..0c918a3 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -48,6 +48,7 @@ SPEC_OWNED = { } STANDALONE = { + "push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs", "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", diff --git a/src/openultrasast/push_inputs.py b/src/openultrasast/push_inputs.py new file mode 100644 index 0000000..7f6190d --- /dev/null +++ b/src/openultrasast/push_inputs.py @@ -0,0 +1,158 @@ +"""Offline-first provenance checks for declared push evaluation populations. + +This validates inputs, never detector outcomes or capability admission. Source bytes come +from pinned Git blobs, so a dirty cached worktree cannot silently change the experiment. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +from dataclasses import replace +from pathlib import Path, PurePosixPath +from typing import Any + +from openultrasast.repos import RepoUnavailable, load_repo_recipe, resolve + + +class InputError(ValueError): + """An input identity or label cannot be trusted.""" + + +def _blob(root: Path, commit: str, path: str) -> bytes: + pure = PurePosixPath(path) + if pure.is_absolute() or ".." in pure.parts or not path or "\\" in path: + raise InputError(f"unsafe source path: {path!r}") + # Git object reads neither follow worktree symlinks nor execute project code. + result = subprocess.run( + ["git", "-C", str(root), "cat-file", "blob", f"{commit}:{path}"], + capture_output=True, + check=True, + timeout=30, + ) + if not result.stdout: + raise InputError(f"empty source bytes: {path}") + return result.stdout + + +def _snapshot(item: dict[str, Any], directory: Path, cache_root: Path | None, fetch: bool) -> dict[str, Any]: + result: dict[str, Any] = {"id": item["id"], "status": "invalid_input", "bytes_read": 0, "files": []} + try: + recipe = load_repo_recipe(directory / item["recipe"]) + commit = item.get("commit", recipe.commit) + if not isinstance(commit, str) or not re.fullmatch("[0-9a-f]{40}", commit): + raise InputError("commit must be a full immutable SHA") + recipe = replace(recipe, commit=commit) + root = resolve(recipe, fetch=fetch, cache_root=cache_root) + result.update(commit=commit, repository=recipe.url, license=recipe.license) + files = item["files"] + paths = [entry["path"] for entry in files] + if recipe.license_file not in paths or len(paths) < 2 or len(paths) != len(set(paths)): + raise InputError("declare a unique license file and at least one source file") + for entry in files: + content = _blob(root, commit, entry["path"]) + result["bytes_read"] += len(content) + edit = entry.get("edit") + if edit: + if not isinstance(edit, dict): + raise InputError("authored edit must be an object") + if not edit.get("provenance", "").startswith("authored"): + raise InputError("transformation must identify authored provenance") + before, after = edit["before"].encode(), edit["after"].encode() + if not before or content.count(before) != 1 or before == after: + raise InputError(f"authored edit must match exactly once: {entry['path']}") + if hashlib.sha256(content).hexdigest() != entry.get("original_sha256"): + raise InputError(f"original hash mismatch: {entry['path']}") + content = content.replace(before, after, 1) + digest = hashlib.sha256(content).hexdigest() + if len(content) != entry["size"] or digest != entry["sha256"] or not content: + raise InputError(f"source size/hash mismatch: {entry['path']}") + result["files"].append({"path": entry["path"], "size": len(content), "sha256": digest}) + result["status"] = "ready" + except RepoUnavailable: + result.update( + status="missing_prerequisite", + problem=f"{recipe.name}: pinned checkout {commit} unavailable; rerun this validator with --fetch to fetch it", + ) + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as exc: + result["problem"] = str(exc) + return result + + +def validate_manifest(path: Path, *, cache_root: Path | None = None, fetch: bool = False) -> dict[str, Any]: + """Retain every case, marking unavailable inputs and unreviewed labels explicitly. + + `ready` means input provenance is verified. It never means a finding was reproduced. + Explicit --fetch is required even if the ordinary repository network env var is set. + """ + manifest_bytes = path.read_bytes() + data = json.loads(manifest_bytes) + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise InputError("unsupported push input schema version") + for field in ("snapshots", "cases"): + if not isinstance(data.get(field), list) or not data[field]: + raise InputError(f"{field} must be a nonempty list") + for item in data[field]: + if not isinstance(item, dict) or not isinstance(item.get("id"), str) or not item["id"]: + raise InputError(f"{field} entry requires an identity") + snapshots = [_snapshot(item, path.parent, cache_root, fetch) for item in data["snapshots"]] + indexed = {item["id"]: item for item in snapshots} + if len(indexed) != len(snapshots): + raise InputError("duplicate snapshot identity") + cases = [] + ids: set[str] = set() + for item in data["cases"]: + case = dict(item, status="invalid_input", admission="experimental") + try: + if item["id"] in ids: + raise InputError("duplicate case identity") + ids.add(item["id"]) + if item["label"] not in {"vulnerable", "fixed", "benign", "regression", "unsupported", "unreviewed"}: + raise InputError("invalid case label") + if item["role"] not in {"development", "transfer", "regression", "envelope", "reserved_holdout"}: + raise InputError("invalid workload role") + if not item["capability"] or not item["provenance"]: + raise InputError("capability and provenance are required") + if item["label"] == "unreviewed" and not item["prerequisites"]: + raise InputError("unreviewed labels require an explicit prerequisite") + inputs = [indexed[item[key]] for key in ("base", "tip")] + problems = [value.get("problem", "") for value in inputs if value["status"] != "ready"] + if any(value["status"] == "invalid_input" for value in inputs): + case["problem"] = "; ".join(problems) + elif problems or item["prerequisites"]: + case.update(status="missing_prerequisite", problem="; ".join(problems + item["prerequisites"])) + else: + case["status"] = "ready" + except (ValueError, KeyError, TypeError) as exc: + case["problem"] = str(exc) + cases.append(case) + return { + "schema_version": 1, + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "population": len(cases), + "capability_prerequisites": data.get("capability_prerequisites", {}), + "snapshots": snapshots, + "cases": cases, + "note": "Input validation only; every capability remains experimental pending independent evaluation.", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("--fetch", action="store_true") + args = parser.parse_args() + try: + result = validate_manifest(args.manifest, fetch=args.fetch) + except (ValueError, OSError) as exc: + print(json.dumps({"status": "invalid_input", "problem": str(exc)})) + return 2 + print(json.dumps(result, indent=2)) + return 0 if all(case["status"] == "ready" for case in result["cases"]) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_push_inputs.py b/tests/test_push_inputs.py new file mode 100644 index 0000000..c28da8a --- /dev/null +++ b/tests/test_push_inputs.py @@ -0,0 +1,141 @@ +"""Input provenance fails closed without dropping a declared evaluation case.""" + +import hashlib +import json +import subprocess + +import pytest + +from openultrasast.push_inputs import validate_manifest + + +def git(root, *args): + return subprocess.check_output(["git", "-C", str(root), *args]).decode().strip() + + +@pytest.fixture +def manifest(tmp_path): + source = tmp_path / "repo" + source.mkdir() + git(source, "init", "-q") + for name, value in [("LICENSE", "test license\n"), ("api.js", "eval(req.body.value);\n")]: + (source / name).write_text(value) + git(source, "add", ".") + git(source, "-c", "user.name=Test", "-c", "user.email=test@example.org", "commit", "-qm", "input") + sha = git(source, "rev-parse", "HEAD") + cache = tmp_path / "cache" / "sample" / sha[:12] + cache.parent.mkdir(parents=True) + source.rename(cache) + recipe = tmp_path / "sample.toml" + recipe.write_text( + f'name="sample"\nurl="https://example.org/sample.git"\ncommit="{sha}"\nlanguage="javascript"\nlicense="MIT"\nlicense_file="LICENSE"\nmeasures=["envelope"]\n' + ) + files = [ + {"path": p.name, "sha256": hashlib.sha256(p.read_bytes()).hexdigest(), "size": p.stat().st_size} + for p in [cache / "LICENSE", cache / "api.js"] + ] + data = { + "schema_version": 1, + "snapshots": [{"id": "vuln", "recipe": "sample.toml", "files": files}], + "cases": [ + { + "id": "case", + "base": "vuln", + "tip": "vuln", + "label": "vulnerable", + "role": "development", + "capability": "javascript/express/injection", + "provenance": "upstream teaching example", + "prerequisites": [], + } + ], + } + path = tmp_path / "inputs.json" + + def run(): + path.write_text(json.dumps(data)) + return validate_manifest(path, cache_root=tmp_path / "cache") + + return data, cache, run + + +def test_immutable_bytes_ignore_dirty_worktree(manifest): + _, root, run = manifest + (root / "api.js").write_text("different worktree\n") + result = run() + assert result["cases"][0]["status"] == "ready" + assert result["snapshots"][0]["bytes_read"] > 0 + + +@pytest.mark.parametrize("mutation", ["hash", "empty", "escape", "missing", "ref"]) +def test_invalid_input_keeps_case_in_population(manifest, mutation): + data, _, run = manifest + snap = data["snapshots"][0] + file = snap["files"][1] + if mutation == "hash": + file["sha256"] = "0" * 64 + elif mutation == "empty": + file["size"] = 0 + elif mutation == "escape": + file["path"] = "../api.js" + elif mutation == "missing": + file["path"] = "missing.js" + else: + snap["commit"] = "main" + result = run() + assert result["population"] == 1 + assert result["cases"][0]["status"] == "invalid_input" + + +def test_unavailable_source_is_explicit(manifest): + data, _, run = manifest + data["snapshots"][0]["commit"] = "a" * 40 + result = run() + assert result["cases"][0]["status"] == "missing_prerequisite" + assert "fetch" in str(result) + + +def test_missing_holdout_labels_never_qualify_capability(manifest): + data, _, run = manifest + data["cases"][0].update(role="reserved_holdout", label="unreviewed", prerequisites=["review independent positive/fixed/benign pair"]) + result = run() + assert result["cases"][0]["status"] == "missing_prerequisite" + assert result["cases"][0]["admission"] == "experimental" + + +def test_invalid_label_rejected(manifest): + data, _, run = manifest + data["cases"][0]["label"] = "clean" + assert run()["cases"][0]["status"] == "invalid_input" + + +def test_authored_edit_requires_exact_original_and_provenance(manifest): + data, _, run = manifest + f = data["snapshots"][0]["files"][1] + f["edit"] = {"before": "not in source", "after": "safe", "provenance": "authored repair"} + assert run()["cases"][0]["status"] == "invalid_input" + + +@pytest.mark.parametrize("field", ["snapshots", "cases"]) +def test_empty_population_rejected(manifest, field): + from openultrasast.push_inputs import InputError + + data, _, run = manifest + data[field] = [] + with pytest.raises(InputError, match="nonempty"): + run() + + +def test_missing_snapshot_identity_rejected(manifest): + from openultrasast.push_inputs import InputError + + data, _, run = manifest + del data["snapshots"][0]["id"] + with pytest.raises(InputError, match="identity"): + run() + + +def test_missing_language_holdouts_retained(manifest): + data, _, run = manifest + data["capability_prerequisites"] = {"php/wordpress/injection": ["untouched PHP pair missing"]} + assert run()["capability_prerequisites"] == data["capability_prerequisites"]