From d0e0ecad78f9602ddb5577fd0705105e5a5ac8a8 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 13 Sep 2026 12:56:55 +0200 Subject: [PATCH] feat(pre-push-safety-net): admit only supported actionable changes --- .kiro/specs/pre-push-safety-net/design.md | 10 + .../implementation-group-4.md | 23 + .kiro/specs/pre-push-safety-net/spec.json | 5 +- .kiro/specs/pre-push-safety-net/tasks.md | 5 +- .../2026-09-13-admission-policy-smoke.json | 930 ++++++++++++++++++ ops/README.md | 6 + ops/smoke_admission.py | 68 ++ src/openultrasast/push/policy.py | 253 ++++- tests/test_push_admission.py | 223 +++++ 9 files changed, 1516 insertions(+), 7 deletions(-) create mode 100644 benchmarks/measurements/2026-09-13-admission-policy-smoke.json create mode 100644 ops/smoke_admission.py create mode 100644 tests/test_push_admission.py diff --git a/.kiro/specs/pre-push-safety-net/design.md b/.kiro/specs/pre-push-safety-net/design.md index 7102f9f..7244621 100644 --- a/.kiro/specs/pre-push-safety-net/design.md +++ b/.kiro/specs/pre-push-safety-net/design.md @@ -308,6 +308,16 @@ patch generation. Templates grounded in facts/witnesses are sufficient. Admissio "sanitize input" advice without a supported context/operation. An LLM explanation must cite the same witness and cannot raise the rung or invent exploitability. +Task 4.2 binds each delta to its base revision, head revision and semantics identity; +legacy unbound evidence remains readable but cannot be admitted. Eligibility declarations +are trusted outputs of the later evaluation loader, with positive/reviewed populations, +calibration and untouched-workload provenance. The default registry is empty. Each +declaration narrows its semantic context to exact operation symbols, preserving receiver +spelling; a broad family such as injection cannot select shell advice for a SQL operation. +Unknown or dynamic spellings remain ineligible. This is admission over existing graph +witnesses, not a new detector or an alias resolver. Per-candidate records retain the full +matching evaluation declarations and exact comparison/ref ownership. + ### Result and Hook Adapter `PushResult` separates `finding_status` (`actionable` / `none`), `coverage_status` (`complete_within_scope` diff --git a/.kiro/specs/pre-push-safety-net/implementation-group-4.md b/.kiro/specs/pre-push-safety-net/implementation-group-4.md index e6f5266..6137db8 100644 --- a/.kiro/specs/pre-push-safety-net/implementation-group-4.md +++ b/.kiro/specs/pre-push-safety-net/implementation-group-4.md @@ -39,3 +39,26 @@ limitation for the later replay integration. These controls use declared questio and Git context directly; no missing-coverage result was erased. Ambiguous source and identifier changes remain unknown. Task 4.1 brings progress to 12/27 tasks; admission and rendering remain pending within this authorized group. + +## Task 4.2 verification + +VERIFIED (TASK scope), independent review APPROVED. Full suite: 1,207 passed, +nine skipped in 99.04 seconds; global Ruff/format (235 files), mypy (110 source +files), detection/map/pair gates and diff checks passed. OFF admission controls +failed before implementation. Review reproduced a SQL witness acquiring shell advice +through a broad injection-family key; evaluated operation-symbol restrictions now +reject that mismatch, missing bindings, receiver changes and embedded-name matches. + +Deltas bind exact base/head and semantics; unbound legacy records cannot be admitted. +Evaluation declarations retain calibration and untouched-population provenance and +must supply positive controls and reviewed alerts. Every disposition preserves its +evidence and matching declarations. Deduplication retains witnesses, locations, +refs and exact comparisons; unchanged debt on one comparison cannot erase a new +claim on another. Advisory and blocking share admission; coverage policy is separate. + +`benchmarks/measurements/2026-09-13-admission-policy-smoke.json` records current +policy against captured actual engine answers: all remain diagnostic with the empty +default registry. Explicitly synthetic PHP/WordPress and JavaScript/Express controls +receive equivalent admission and enforcement behavior. They do not qualify any +real capability; task 8.3 still owns actual eligibility. Task 4.2 brings progress to +13/27 tasks. Compact reporting remains in this authorized group. diff --git a/.kiro/specs/pre-push-safety-net/spec.json b/.kiro/specs/pre-push-safety-net/spec.json index f82cc4a..5c9ee51 100644 --- a/.kiro/specs/pre-push-safety-net/spec.json +++ b/.kiro/specs/pre-push-safety-net/spec.json @@ -84,9 +84,10 @@ "3.2", "3.3", "3.4", - "4.1" + "4.1", + "4.2" ], - "remaining_tasks": 15, + "remaining_tasks": 14, "verification_record": "implementation-group-4.md", "verification_records": [ "implementation-group-1.md", diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index fee951d..a4b0f39 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -1,7 +1,7 @@ # Implementation Plan: pre-push-safety-net Status: task plan approved by the maintainer's "approve." on 2026-09-12. Groups 1–3 (tasks 1.1–3.4) -are complete after independent review and fresh verification (12/27 subtasks, including task 4.1). Tasks 4.2–4.3 and groups 5–8 remain pending; +are complete after independent review and fresh verification (13/27 subtasks, including tasks 4.1–4.2). Task 4.3 and groups 5–8 remain pending; the hook and capability admission are not yet implemented. See `implementation-group-1.md` and `implementation-group-2.md`. ## Execution contract @@ -121,12 +121,13 @@ the hook and capability admission are not yet implemented. See `implementation-g - _Boundary: Delta and Actionability Policy, engine comparison integration_ - _Depends: 2.3, 3.3, 3.4_ - _Requirements: 2.4, 3.1, 3.4, 4.1_ -- [ ] 4.2 Enforce capability and actionability admission +- [x] 4.2 Enforce capability and actionability admission - Require evaluated capability provenance, applicable semantics, a sufficient rung, exact witness/location, supported change attribution and a concrete repair direction. - Keep experimental, suspicious, wrongly sanitized and unknown-novelty candidates in diagnostic records; do not let model confidence, a sanitizer name or printed-before state admit/dismiss a defect. - Deduplicate by defect while retaining relevant witnesses and refs; distinguish experimental evaluation from normal hook eligibility. - Add equivalent PHP/JavaScript normalized claims with equally eligible capabilities and assert equal admission; add an unsupported-origin control whose coverage/eligibility differs without changing core scores. - Done when the policy rejects an entailed but unsupported claim and admits fully supported synthetic controls equivalently across origins and advisory/blocking modes; real capabilities remain disabled pending 8.3. + - _Verified 2026-09-13:_ independent review APPROVED after operation/context-binding repair; 1,207 tests passed, nine skipped; default real-evidence controls remain diagnostic and synthetic PHP/JavaScript admission is equivalent. See `implementation-group-4.md`. - _Boundary: Delta and Actionability Policy_ - _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5, 5.3, 5.4, 8.1_ - [ ] 4.3 Render compact results and preserve complete artifacts diff --git a/benchmarks/measurements/2026-09-13-admission-policy-smoke.json b/benchmarks/measurements/2026-09-13-admission-policy-smoke.json new file mode 100644 index 0000000..0d45227 --- /dev/null +++ b/benchmarks/measurements/2026-09-13-admission-policy-smoke.json @@ -0,0 +1,930 @@ +{ + "status": "VERIFIED", + "task": "4.2", + "purpose": "Synthetic eligibility controls and current policy on captured actual evidence; no real capability enabled", + "source_hashes": { + "src/openultrasast/push/policy.py": "bbe1b5d0bd8a50473996ba2f0c25649b11b58c5daee02afd54bdd95b370f7be2", + "tests/test_push_admission.py": "6e96faa62b8838a8d2d85096f8e8740f570003be29ad0f0c144e4c7bea145c4d", + "ops/smoke_admission.py": "79b5270de654708cae6aa53f4c93038aed4e7c6a9a4451a9970d4f97e24cd766" + }, + "default_registry_actual_evidence": { + "purpose": "Current policy applied to captured actual engine evidence; no new engine execution or evaluated capability", + "input_bytes": 44685, + "input_sha256": "40c99b72238526b34ca7dc78fcae74c88e05ed9aa6ea2c5e66cdb4be0d14cebd", + "cases": { + "guard": { + "admission": { + "defects": [], + "dispositions": [ + { + "candidate": { + "delta": { + "family": "access_control", + "site": "api.js:2:handler", + "rung": "model_entailed", + "witness": "handler line 2: no discharging guard, while 1 sibling handler(s) are guarded (sibling)", + "defect_id": "c3c274c9b04b03b2df9ef8090ab2ca7ecbaafeea1b2b5f353972cb2168e836fc", + "novelty": "worsened", + "reason": "discharge_removed", + "head_operation": { + "question": { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + }, + "mechanism": "dominance", + "path": "api.js", + "line": 2, + "operation": "deleteUser(req.params.id)", + "source": "obligation", + "discharged": false, + "detail": "{\"dominatingGuards\": [], \"opFile\": \"api.js\", \"opLine\": \"2\", \"opMethod\": \"handler\", \"operation\": \"deleteUser(req.params.id)\"}" + }, + "base_operations": [ + { + "question": { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + }, + "mechanism": "dominance", + "path": "api.js", + "line": 3, + "operation": "deleteUser(req.params.id)", + "source": "obligation", + "discharged": true, + "detail": "{\"dominatingGuards\": [\"!authenticate(req)\", \"authenticate(req)\"], \"opFile\": \"api.js\", \"opLine\": \"3\", \"opMethod\": \"handler\", \"operation\": \"deleteUser(req.params.id)\"}" + } + ], + "change_evidence": [ + "base:api.js:2-2" + ], + "base_revision": "50dd14003e143de73eceda7b45e91ef3490e8d19", + "head_revision": "3c0c7367d68647862b77f7c5fd5a5b8a28ebfad7", + "semantics": "2417155d31b4a5b3ea9ea282efedf23ab72fd382e757ea27e4dfa151195f6f3d" + }, + "capability": { + "language": "javascript", + "runtime": "unspecified", + "framework": "unspecified", + "family": "access_control", + "mechanism": "dominance", + "context": "unreviewed", + "semantics": "2417155d31b4a5b3ea9ea282efedf23ab72fd382e757ea27e4dfa151195f6f3d" + }, + "comparison": { + "head_oid": "3c0c7367d68647862b77f7c5fd5a5b8a28ebfad7", + "base_oid": "50dd14003e143de73eceda7b45e91ef3490e8d19", + "base_reason": "supplied_remote_tip", + "refs": [ + "refs/heads/topic" + ] + }, + "dependency_gaps": [], + "sanitizer_contexts": [] + }, + "admitted": false, + "reasons": [ + "capability_unavailable" + ], + "evaluation_id": null, + "capability_evaluations": [] + } + ], + "coverage_reasons": [ + "capability_unavailable" + ] + }, + "result": { + "analyses": [ + { + "comparison": { + "head_oid": "3c0c7367d68647862b77f7c5fd5a5b8a28ebfad7", + "base_oid": "50dd14003e143de73eceda7b45e91ef3490e8d19", + "base_reason": "supplied_remote_tip", + "refs": [ + "refs/heads/topic" + ] + }, + "scopes": [ + { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "taint_evidence_not_applicable" + ], + "tier": null, + "score": null + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "none", + "coverage_status": "incomplete", + "push_disposition": "allow", + "actionable_defect_ids": [] + } + }, + "movement": { + "admission": { + "defects": [], + "dispositions": [ + { + "candidate": { + "delta": { + "family": "access_control", + "site": "api.js:4:handler", + "rung": "model_entailed", + "witness": "handler line 4: no discharging guard, while 1 sibling handler(s) are guarded (sibling)", + "defect_id": "dd03c2c56c9f9d19721a2caafd8879ab8174fd5078eaf1428ce484388f07675e", + "novelty": "unchanged", + "reason": "same_supported_mechanism", + "head_operation": { + "question": { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + }, + "mechanism": "dominance", + "path": "api.js", + "line": 4, + "operation": "deleteUser(req.params.id)", + "source": "obligation", + "discharged": false, + "detail": "{\"dominatingGuards\": [], \"opFile\": \"api.js\", \"opLine\": \"4\", \"opMethod\": \"handler\", \"operation\": \"deleteUser(req.params.id)\"}" + }, + "base_operations": [ + { + "question": { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + }, + "mechanism": "dominance", + "path": "api.js", + "line": 2, + "operation": "deleteUser(req.params.id)", + "source": "obligation", + "discharged": false, + "detail": "{\"dominatingGuards\": [], \"opFile\": \"api.js\", \"opLine\": \"2\", \"opMethod\": \"handler\", \"operation\": \"deleteUser(req.params.id)\"}" + } + ], + "change_evidence": [ + "head:api.js:1-2" + ], + "base_revision": "ec81bee36092a3fee9f6dab9ab52f2301672197f", + "head_revision": "4b6ff24c4add7a4d5e61c852a8d5e6475734334b", + "semantics": "2417155d31b4a5b3ea9ea282efedf23ab72fd382e757ea27e4dfa151195f6f3d" + }, + "capability": { + "language": "javascript", + "runtime": "unspecified", + "framework": "unspecified", + "family": "access_control", + "mechanism": "dominance", + "context": "unreviewed", + "semantics": "2417155d31b4a5b3ea9ea282efedf23ab72fd382e757ea27e4dfa151195f6f3d" + }, + "comparison": { + "head_oid": "4b6ff24c4add7a4d5e61c852a8d5e6475734334b", + "base_oid": "ec81bee36092a3fee9f6dab9ab52f2301672197f", + "base_reason": "supplied_remote_tip", + "refs": [ + "refs/heads/topic" + ] + }, + "dependency_gaps": [], + "sanitizer_contexts": [] + }, + "admitted": false, + "reasons": [ + "capability_unavailable", + "unchanged" + ], + "evaluation_id": null, + "capability_evaluations": [] + } + ], + "coverage_reasons": [ + "capability_unavailable" + ] + }, + "result": { + "analyses": [ + { + "comparison": { + "head_oid": "4b6ff24c4add7a4d5e61c852a8d5e6475734334b", + "base_oid": "ec81bee36092a3fee9f6dab9ab52f2301672197f", + "base_reason": "supplied_remote_tip", + "refs": [ + "refs/heads/topic" + ] + }, + "scopes": [ + { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "taint_evidence_not_applicable" + ], + "tier": null, + "score": null + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "delta-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "access_control" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "none", + "coverage_status": "incomplete", + "push_disposition": "allow", + "actionable_defect_ids": [] + } + }, + "connection": { + "admission": { + "defects": [], + "dispositions": [ + { + "candidate": { + "delta": { + "family": "injection", + "site": "api.php:4:handler", + "rung": "model_entailed", + "witness": "$req -> eval($value) in api.php (line 4, 4 steps)", + "defect_id": "126c32bef15eed9f5bdbb01c8c582ff03654ca835a61af4bc32b1b5904b7aa34", + "novelty": "new", + "reason": "source_connection_absent_from_comparable_base", + "head_operation": { + "question": { + "unit": "delta-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + }, + "mechanism": "taint", + "path": "api.php", + "line": 4, + "operation": "eval($value)", + "source": "[\"parameter\",\"$req\"]", + "discharged": false, + "detail": "{\"bound\": \"\", \"bounded\": false, \"inLabeledScope\": true, \"length\": 4, \"paths\": 1, \"sanitized\": false, \"sink\": \"eval($value)\", \"sinkArg0Literal\": false, \"sinkArity\": 1, \"sinkFile\": \"api.php\", \"sinkLine\": \"4\", \"sinkMethod\": \"handler\", \"source\": \"$req\", \"sourceKind\": \"parameter\"}" + }, + "base_operations": [], + "change_evidence": [ + "base:api.php:3-3", + "head:api.php:3-3" + ], + "base_revision": "761a1c9f1e891465ba4db952e975d4cc31d43674", + "head_revision": "961b8f28da002916f02864f4d0dbe06af8ad6088", + "semantics": "2417155d31b4a5b3ea9ea282efedf23ab72fd382e757ea27e4dfa151195f6f3d" + }, + "capability": { + "language": "php", + "runtime": "unspecified", + "framework": "unspecified", + "family": "injection", + "mechanism": "taint", + "context": "unreviewed", + "semantics": "2417155d31b4a5b3ea9ea282efedf23ab72fd382e757ea27e4dfa151195f6f3d" + }, + "comparison": { + "head_oid": "961b8f28da002916f02864f4d0dbe06af8ad6088", + "base_oid": "761a1c9f1e891465ba4db952e975d4cc31d43674", + "base_reason": "supplied_remote_tip", + "refs": [ + "refs/heads/topic" + ] + }, + "dependency_gaps": [], + "sanitizer_contexts": [] + }, + "admitted": false, + "reasons": [ + "capability_unavailable" + ], + "evaluation_id": null, + "capability_evaluations": [] + } + ], + "coverage_reasons": [ + "capability_unavailable" + ] + }, + "result": { + "analyses": [ + { + "comparison": { + "head_oid": "961b8f28da002916f02864f4d0dbe06af8ad6088", + "base_oid": "761a1c9f1e891465ba4db952e975d4cc31d43674", + "base_reason": "supplied_remote_tip", + "refs": [ + "refs/heads/topic" + ] + }, + "scopes": [ + { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "delta-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"mysqli_prepare\",\"mysqli_stmt_bind_param\",\"pg_execute\",\"pg_prepare\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[{\"arg0_literal\":false,\"arity\":1,\"cleansed_on_call\":false,\"code\":\"eval($value)\",\"line\":4,\"method\":\"handler\"}],\"source_local\":false,\"source_near\":false}", + "tier=4", + "score=2.5" + ], + "tier": 4, + "score": 2.5 + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "delta-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "none", + "coverage_status": "incomplete", + "push_disposition": "allow", + "actionable_defect_ids": [] + } + } + } + }, + "synthetic_controls": { + "php": { + "label": "Synthetic normalized policy control; eligibility is supplied only for this isolated verification", + "admission": { + "defects": [ + { + "defect_id": "defect", + "family": "injection", + "witnesses": [ + "request.input -> exec(command)" + ], + "locations": [ + [ + "handler.src", + 4 + ] + ], + "comparisons": [ + { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + } + ], + "refs": [ + "refs/heads/main" + ], + "change_evidence": [ + "head:handler.src:2-2" + ], + "consequences": [ + "Untrusted input reaching exec(command) permits command execution in shell-command." + ], + "repairs": [ + "handler.src:4: At exec(command), use an argument-vector process API without a shell for shell-command." + ], + "evaluation_ids": [ + "synthetic-only-evaluation" + ] + } + ], + "dispositions": [ + { + "candidate": { + "delta": { + "family": "injection", + "site": "handler.src:4:handler", + "rung": "model_entailed", + "witness": "request.input -> exec(command)", + "defect_id": "defect", + "novelty": "new", + "reason": "source_connection_absent_from_comparable_base", + "head_operation": { + "question": { + "unit": "repository", + "language": "php", + "path": "handler.src", + "function": "handler", + "family": "injection" + }, + "mechanism": "taint", + "path": "handler.src", + "line": 4, + "operation": "exec(command)", + "source": "request.input", + "discharged": false, + "detail": "{}" + }, + "base_operations": [], + "change_evidence": [ + "head:handler.src:2-2" + ], + "base_revision": "base", + "head_revision": "head", + "semantics": "synthetic-semantics-v1" + }, + "capability": { + "language": "php", + "runtime": "server", + "framework": "wordpress", + "family": "injection", + "mechanism": "taint", + "context": "shell-command", + "semantics": "synthetic-semantics-v1" + }, + "comparison": { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + }, + "dependency_gaps": [], + "sanitizer_contexts": [] + }, + "admitted": true, + "reasons": [], + "evaluation_id": "synthetic-only-evaluation", + "capability_evaluations": [ + { + "key": { + "language": "php", + "runtime": "server", + "framework": "wordpress", + "family": "injection", + "mechanism": "taint", + "context": "shell-command", + "semantics": "synthetic-semantics-v1" + }, + "evaluation_id": "synthetic-only-evaluation", + "evaluation_artifact": "synthetic://reviewed-controls", + "verdict": "PASS", + "consequence_template": "Untrusted input reaching {operation} permits command execution in {context}.", + "repair_template": "At {operation}, use an argument-vector process API without a shell for {context}.", + "enabled": true, + "positive_controls": 2, + "reviewed_alerts": 2, + "calibration_artifact": "synthetic://scorecard", + "untouched_workload": "synthetic-control-population", + "operation_symbols": [ + "exec" + ] + } + ] + } + ], + "coverage_reasons": [] + }, + "results": { + "advisory": { + "analyses": [ + { + "comparison": { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + }, + "scopes": [ + { + "ranker_version": "v1", + "ranking_mode": "static", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "repository", + "language": "php", + "path": "handler.src", + "function": "handler", + "family": "injection" + }, + "priority": 1.0, + "evidence": [], + "tier": null, + "score": null + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "repository", + "language": "php", + "path": "handler.src", + "function": "handler", + "family": "injection" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "actionable", + "coverage_status": "complete_within_scope", + "push_disposition": "allow", + "actionable_defect_ids": [ + "defect" + ] + }, + "blocking": { + "analyses": [ + { + "comparison": { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + }, + "scopes": [ + { + "ranker_version": "v1", + "ranking_mode": "static", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "repository", + "language": "php", + "path": "handler.src", + "function": "handler", + "family": "injection" + }, + "priority": 1.0, + "evidence": [], + "tier": null, + "score": null + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "repository", + "language": "php", + "path": "handler.src", + "function": "handler", + "family": "injection" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "actionable", + "coverage_status": "complete_within_scope", + "push_disposition": "block", + "actionable_defect_ids": [ + "defect" + ] + } + } + }, + "javascript": { + "label": "Synthetic normalized policy control; eligibility is supplied only for this isolated verification", + "admission": { + "defects": [ + { + "defect_id": "defect", + "family": "injection", + "witnesses": [ + "request.input -> exec(command)" + ], + "locations": [ + [ + "handler.src", + 4 + ] + ], + "comparisons": [ + { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + } + ], + "refs": [ + "refs/heads/main" + ], + "change_evidence": [ + "head:handler.src:2-2" + ], + "consequences": [ + "Untrusted input reaching exec(command) permits command execution in shell-command." + ], + "repairs": [ + "handler.src:4: At exec(command), use an argument-vector process API without a shell for shell-command." + ], + "evaluation_ids": [ + "synthetic-only-evaluation" + ] + } + ], + "dispositions": [ + { + "candidate": { + "delta": { + "family": "injection", + "site": "handler.src:4:handler", + "rung": "model_entailed", + "witness": "request.input -> exec(command)", + "defect_id": "defect", + "novelty": "new", + "reason": "source_connection_absent_from_comparable_base", + "head_operation": { + "question": { + "unit": "repository", + "language": "javascript", + "path": "handler.src", + "function": "handler", + "family": "injection" + }, + "mechanism": "taint", + "path": "handler.src", + "line": 4, + "operation": "exec(command)", + "source": "request.input", + "discharged": false, + "detail": "{}" + }, + "base_operations": [], + "change_evidence": [ + "head:handler.src:2-2" + ], + "base_revision": "base", + "head_revision": "head", + "semantics": "synthetic-semantics-v1" + }, + "capability": { + "language": "javascript", + "runtime": "server", + "framework": "express", + "family": "injection", + "mechanism": "taint", + "context": "shell-command", + "semantics": "synthetic-semantics-v1" + }, + "comparison": { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + }, + "dependency_gaps": [], + "sanitizer_contexts": [] + }, + "admitted": true, + "reasons": [], + "evaluation_id": "synthetic-only-evaluation", + "capability_evaluations": [ + { + "key": { + "language": "javascript", + "runtime": "server", + "framework": "express", + "family": "injection", + "mechanism": "taint", + "context": "shell-command", + "semantics": "synthetic-semantics-v1" + }, + "evaluation_id": "synthetic-only-evaluation", + "evaluation_artifact": "synthetic://reviewed-controls", + "verdict": "PASS", + "consequence_template": "Untrusted input reaching {operation} permits command execution in {context}.", + "repair_template": "At {operation}, use an argument-vector process API without a shell for {context}.", + "enabled": true, + "positive_controls": 2, + "reviewed_alerts": 2, + "calibration_artifact": "synthetic://scorecard", + "untouched_workload": "synthetic-control-population", + "operation_symbols": [ + "exec" + ] + } + ] + } + ], + "coverage_reasons": [] + }, + "results": { + "advisory": { + "analyses": [ + { + "comparison": { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + }, + "scopes": [ + { + "ranker_version": "v1", + "ranking_mode": "static", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "repository", + "language": "javascript", + "path": "handler.src", + "function": "handler", + "family": "injection" + }, + "priority": 1.0, + "evidence": [], + "tier": null, + "score": null + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "repository", + "language": "javascript", + "path": "handler.src", + "function": "handler", + "family": "injection" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "actionable", + "coverage_status": "complete_within_scope", + "push_disposition": "allow", + "actionable_defect_ids": [ + "defect" + ] + }, + "blocking": { + "analyses": [ + { + "comparison": { + "head_oid": "head", + "base_oid": "base", + "base_reason": "remote", + "refs": [ + "refs/heads/main" + ] + }, + "scopes": [ + { + "ranker_version": "v1", + "ranking_mode": "static", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "repository", + "language": "javascript", + "path": "handler.src", + "function": "handler", + "family": "injection" + }, + "priority": 1.0, + "evidence": [], + "tier": null, + "score": null + } + ], + "deferred": [], + "unresolved_boundaries": [] + } + ], + "completed_questions": [ + { + "unit": "repository", + "language": "javascript", + "path": "handler.src", + "function": "handler", + "family": "injection" + } + ], + "coverage_status": "complete_within_scope" + } + ], + "finding_status": "actionable", + "coverage_status": "complete_within_scope", + "push_disposition": "block", + "actionable_defect_ids": [ + "defect" + ] + } + } + } + }, + "claim_type": "TASK", + "verification": { + "review": "APPROVED", + "tests_passed": 1207, + "tests_skipped": 9, + "seconds": 99.04, + "ruff": "PASS", + "format_files": 235, + "mypy_files": 110, + "regression_gates": "PASS" + } +} diff --git a/ops/README.md b/ops/README.md index 5fdad1b..a810353 100644 --- a/ops/README.md +++ b/ops/README.md @@ -214,3 +214,9 @@ state survives. Output contains an evidence JSON object followed by a completion JSON object; success requires exit zero and `verified: true` in that final object. These declared regions exercise comparison mechanics; full replay, normal alert admission and hook latency are separate evaluation work. + +`python ops/smoke_admission.py` reads the committed comparison measurement and +rechecks its actual engine answers through the current policy. With no evaluated +capabilities supplied, every candidate stays diagnostic and the result separates +`none` findings, `incomplete` coverage and `allow` disposition. This offline control +executes no new graph query and does not qualify any capability for normal alerts. diff --git a/ops/smoke_admission.py b/ops/smoke_admission.py new file mode 100644 index 0000000..c58e08d --- /dev/null +++ b/ops/smoke_admission.py @@ -0,0 +1,68 @@ +"""Verify that captured real comparison evidence remains diagnostic by default.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from openultrasast.model.contracts import ChangeContext, QuestionOutcome, ScopeDecision +from openultrasast.model.ladder import Rung +from openultrasast.model.pipeline import ModelFinding +from openultrasast.model.scan import ModelScanResult +from openultrasast.push.contracts import ComparisonAnalysis, PushComparison +from openultrasast.push.policy import AdmissionCandidate, CapabilityKey, admission_push_result, admit_candidates, compare_evidence + + +def restore_scan(data: dict) -> ModelScanResult: + """Restore only the recorded evidence consumed by the comparison policy.""" + return ModelScanResult( + findings=tuple(ModelFinding(**{**finding, "rung": Rung(finding["rung"])}) for finding in data["findings"]), + scope=ScopeDecision.from_payload(data["scope"]), + question_outcomes=tuple(QuestionOutcome.from_payload(outcome) for outcome in data["question_outcomes"]), + degradations=tuple(data["degradations"]), + ) + + +def main() -> None: + artifact = Path("benchmarks/measurements/2026-09-13-delta-comparison-smoke.json") + raw = artifact.read_bytes() + assert raw, "comparison artifact is unreadable or empty" + recorded = json.loads(raw) + report = { + "purpose": "Current policy applied to captured actual engine evidence; no new engine execution or evaluated capability", + "input_bytes": len(raw), + "input_sha256": hashlib.sha256(raw).hexdigest(), + "cases": {}, + } + for case, record in recorded["cases"].items(): + evidence = record["evidence"] + head = restore_scan(evidence["head"]) + base = restore_scan(evidence["delta"]["base_scan"]) + context = ChangeContext.from_payload(evidence["context"]) + comparison = PushComparison.from_payload(evidence["comparison"]) + semantics = evidence["delta"]["semantics"] + deltas = compare_evidence(head, base, context=context, head_semantics=semantics, base_semantics=semantics) + assert deltas and all(delta.novelty == evidence["expected_novelty"] for delta in deltas) + candidates = [] + for delta in deltas: + operation = delta.head_operation + assert operation is not None + key = CapabilityKey( + operation.question.language, "unspecified", "unspecified", delta.family, operation.mechanism, "unreviewed", semantics + ) + candidates.append(AdmissionCandidate(delta, key, comparison)) + admission = admit_candidates(candidates) + assert not admission.defects and len(admission.dispositions) == len(candidates) + assert all(not disposition.admitted for disposition in admission.dispositions) + assert "capability_unavailable" in admission.coverage_reasons + assert head.scope is not None + analysis = ComparisonAnalysis(comparison, (head.scope,), tuple(q.identity for q in head.question_outcomes), "complete_within_scope") + result = admission_push_result(admission, (analysis,)) + assert result.finding_status == "none" and result.coverage_status == "incomplete" and result.push_disposition == "allow" + report["cases"][case] = {"admission": admission.to_payload(), "result": result.to_payload()} + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/openultrasast/push/policy.py b/src/openultrasast/push/policy.py index 21d50c2..bd215e0 100644 --- a/src/openultrasast/push/policy.py +++ b/src/openultrasast/push/policy.py @@ -1,4 +1,4 @@ -"""Compare arbiter-backed evidence from immutable revisions; this does not admit alerts.""" +"""Compare immutable revision evidence and apply evaluated capability admission.""" from __future__ import annotations @@ -7,9 +7,11 @@ import json import time from collections.abc import Sequence from dataclasses import dataclass, replace -from pathlib import Path +from pathlib import Path, PurePosixPath +from string import Formatter from typing import Any, Literal +from openultrasast.config import PushConfig from openultrasast.contracts import Contract from openultrasast.cpg.backend import CpgResult from openultrasast.model.contracts import ChangeContext, ExecutionBudget, QuestionIdentity @@ -18,6 +20,7 @@ from openultrasast.model.pipeline import ModelFinding, _arbitrate_all from openultrasast.model.regions import ScanRegion from openultrasast.model.scan import ModelScanResult, ScanBudget, _prefetched, _spec_for, scan_repository from openultrasast.model.specs import ConfigSpec, DominanceSpec, TaintSpec +from openultrasast.push.contracts import ComparisonAnalysis, PushComparison, PushResult @dataclass(frozen=True) @@ -44,6 +47,9 @@ class CandidateDelta(Contract): head_operation: EvidenceOperation | None base_operations: tuple[EvidenceOperation, ...] change_evidence: tuple[str, ...] + base_revision: str | None = None + head_revision: str | None = None + semantics: str | None = None @dataclass(frozen=True) @@ -311,7 +317,15 @@ def compare_evidence( ): novelty, reason = "new", "source_connection_absent_from_comparable_base" results.append(_candidate(finding, op, old, novelty, reason, change, context)) - return tuple(results) + return tuple( + replace( + candidate, + base_revision=context.base_revision, + head_revision=context.head_revision, + semantics=head_semantics if head_semantics == base_semantics else None, + ) + for candidate in results + ) def compare_targeted_base( @@ -372,3 +386,236 @@ def compare_targeted_base( ) reasons.extend(c.reason for c in candidates if c.novelty == "unknown") return DeltaComparison(candidates, base, head_semantics, tuple(dict.fromkeys(reasons))) + + +@dataclass(frozen=True) +class CapabilityKey(Contract): + language: str + runtime: str + framework: str + family: str + mechanism: Literal["taint", "dominance", "config"] + context: str + semantics: str + + +@dataclass(frozen=True) +class CapabilityAdmission(Contract): + """Evaluated declaration supplied by the eligibility loader (task 8.3). + + This contract is not an evaluator or a signature verifier. No shipped declaration + is enabled; synthetic policy tests cannot qualify a real capability. + Templates belong to the evaluated declaration, never to model-generated prose. + """ + + key: CapabilityKey + evaluation_id: str + evaluation_artifact: str + verdict: Literal["PASS", "NO-GO", "experimental"] + consequence_template: str + repair_template: str + enabled: bool = False + positive_controls: int = 0 + reviewed_alerts: int = 0 + calibration_artifact: str | None = None + untouched_workload: str | None = None + # Exact operation symbols covered by this evaluated semantic context. Receiver + # spelling is preserved; unresolved/dynamic forms do not acquire eligibility. + operation_symbols: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AdmissionCandidate(Contract): + delta: CandidateDelta + capability: CapabilityKey + comparison: PushComparison + dependency_gaps: tuple[str, ...] = () + # Semantic contexts of sanitizer facts used in interpreting this witness. A + # name alone is not evidence that an HTML escape discharges a SQL obligation. + sanitizer_contexts: tuple[str, ...] = () + + +@dataclass(frozen=True) +class CandidateDisposition(Contract): + candidate: AdmissionCandidate + admitted: bool + reasons: tuple[str, ...] + evaluation_id: str | None + capability_evaluations: tuple[CapabilityAdmission, ...] + + +@dataclass(frozen=True) +class ActionableDefect(Contract): + defect_id: str + family: str + witnesses: tuple[str, ...] + locations: tuple[tuple[str, int], ...] + comparisons: tuple[PushComparison, ...] + refs: tuple[str, ...] + change_evidence: tuple[str, ...] + consequences: tuple[str, ...] + repairs: tuple[str, ...] + evaluation_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class AdmissionResult(Contract): + defects: tuple[ActionableDefect, ...] + dispositions: tuple[CandidateDisposition, ...] + coverage_reasons: tuple[str, ...] + + +def _grounded_template(template: str, operation: EvidenceOperation, context: str) -> str | None: + try: + parts = tuple(Formatter().parse(template)) + if {field for _, field, _, _ in parts if field is not None} != {"operation", "context"}: + return None + if any(spec or conversion for _, _, spec, conversion in parts): + return None + return template.format(operation=operation.operation, context=context) + except (ValueError, KeyError, IndexError): + return None + + +def admit_candidates(candidates: Sequence[AdmissionCandidate], *, capabilities: Sequence[CapabilityAdmission] = ()) -> AdmissionResult: + """Apply one origin-agnostic evidence bar for both advisory and blocking. + + Capabilities are trusted evaluated declarations, not user/model assertions. + Task 8.3 owns their generation/loading; the default registry is deliberately empty. + Unknown legacy delta provenance cannot acquire eligibility by being wrapped here. + """ + defects: dict[str, ActionableDefect] = {} + dispositions = [] + gaps: list[str] = [] + for candidate in candidates: + delta, key, op = candidate.delta, candidate.capability, candidate.delta.head_operation + reasons = [] + matching = [cap for cap in capabilities if cap.key == key] + cap = matching[0] if len(matching) == 1 else None + if not matching: + reasons.append("capability_unavailable") + elif len(matching) > 1: + reasons.append("capability_ambiguous") + elif cap is not None: + if not cap.enabled: + reasons.append("capability_disabled") + if ( + cap.verdict != "PASS" + or cap.positive_controls < 1 + or cap.reviewed_alerts < 1 + or cap.calibration_artifact is None + or cap.untouched_workload is None + ): + reasons.append("capability_unevaluated") + if ( + delta.base_revision is None + or delta.head_revision is None + or delta.semantics is None + or delta.base_revision != candidate.comparison.base_oid + or delta.head_revision != candidate.comparison.head_oid + or delta.semantics != key.semantics + ): + reasons.append("comparison_provenance_mismatch") + if delta.rung != Rung.ENTAILED.value: + # No production execution-confirmed producer exists. Do not accept a label + # in anticipation of one; a future producer needs an explicit protocol. + reasons.append("insufficient_rung") + if delta.novelty == "unknown": + reasons.append("comparison_unknown") + elif delta.novelty == "unchanged": + reasons.append("unchanged") + elif not delta.change_evidence or (delta.novelty, delta.reason) not in ( + ("new", "source_connection_absent_from_comparable_base"), + ("worsened", "discharge_removed"), + ): + reasons.append("change_unsupported") + if op is None or delta.witness is None: + reasons.append("witness_unresolved") + consequence = repair = None + if op is not None: + path = PurePosixPath(op.path) + if op.line < 1 or path.is_absolute() or ".." in path.parts or not delta.site.startswith(f"{op.path}:{op.line}:"): + reasons.append("location_unresolved") + if (op.question.language, op.question.family, op.mechanism) != ( + key.language, + key.family, + key.mechanism, + ) or delta.family != key.family: + reasons.append("family_semantics_mismatch") + if op.discharged: + reasons.append("claim_discharged") + if cap is not None: + # These are existing graph operation spellings, not source scanning. + # A declaration may narrow an entailed claim; it cannot create one. + symbol = op.operation.partition("(")[0].strip() + if symbol not in cap.operation_symbols: + reasons.append("operation_semantics_mismatch") + consequence = _grounded_template(cap.consequence_template, op, key.context) + repair = _grounded_template(cap.repair_template, op, key.context) + if consequence is None: + reasons.append("consequence_unsupported") + if repair is None: + reasons.append("repair_unsupported") + if candidate.dependency_gaps: + reasons.append("dependency_unresolved") + if any(context != key.context for context in candidate.sanitizer_contexts): + reasons.append("sanitizer_semantics_mismatch") + previous = defects.get(delta.defect_id) + if previous is not None and previous.family != delta.family: + reasons.append("defect_identity_collision") + dispositions.append( + CandidateDisposition(candidate, not reasons, tuple(reasons), cap.evaluation_id if cap else None, tuple(matching)) + ) + gaps.extend(reason for reason in reasons if reason not in ("unchanged", "insufficient_rung", "claim_discharged")) + if reasons: + continue + assert op is not None and delta.witness is not None and consequence is not None and repair is not None and cap is not None + repair = f"{op.path}:{op.line}: {repair}" + if previous is None: + defects[delta.defect_id] = ActionableDefect( + delta.defect_id, + delta.family, + (delta.witness,), + ((op.path, op.line),), + (candidate.comparison,), + candidate.comparison.refs, + delta.change_evidence, + (consequence,), + (repair,), + (cap.evaluation_id,), + ) + else: + defects[delta.defect_id] = replace( + previous, + witnesses=tuple(dict.fromkeys((*previous.witnesses, delta.witness))), + locations=tuple(dict.fromkeys((*previous.locations, (op.path, op.line)))), + comparisons=tuple(dict.fromkeys((*previous.comparisons, candidate.comparison))), + refs=tuple(dict.fromkeys((*previous.refs, *candidate.comparison.refs))), + change_evidence=tuple(dict.fromkeys((*previous.change_evidence, *delta.change_evidence))), + consequences=tuple(dict.fromkeys((*previous.consequences, consequence))), + repairs=tuple(dict.fromkeys((*previous.repairs, repair))), + evaluation_ids=tuple(dict.fromkeys((*previous.evaluation_ids, cap.evaluation_id))), + ) + return AdmissionResult(tuple(defects.values()), tuple(dispositions), tuple(dict.fromkeys(gaps))) + + +def admission_push_result( + admission: AdmissionResult, analyses: tuple[ComparisonAnalysis, ...], *, config: PushConfig | None = None +) -> PushResult: + """Finding admission and coverage remain independent of push enforcement.""" + config = config or PushConfig() + comparisons = {analysis.comparison for analysis in analyses} + if any(comparison not in comparisons for defect in admission.defects for comparison in defect.comparisons): + raise ValueError("admitted defect requires its exact comparison analysis") + coverage: Literal["complete_within_scope", "incomplete", "unavailable", "not_applicable"] + if not analyses or all(analysis.coverage_status == "unavailable" for analysis in analyses): + coverage = "unavailable" + elif admission.coverage_reasons or any(analysis.coverage_status != "complete_within_scope" for analysis in analyses): + coverage = "incomplete" + else: + coverage = "complete_within_scope" + ids = tuple(defect.defect_id for defect in admission.defects) + block = config.mode == "blocking" and ( + bool(ids) or (coverage in ("incomplete", "unavailable") and config.incomplete_coverage_policy == "block") + ) + return PushResult(analyses, "actionable" if ids else "none", coverage, "block" if block else "allow", ids) diff --git a/tests/test_push_admission.py b/tests/test_push_admission.py new file mode 100644 index 0000000..ba9f8ad --- /dev/null +++ b/tests/test_push_admission.py @@ -0,0 +1,223 @@ +"""Synthetic admission controls, not evidence of real capability eligibility.""" + +from dataclasses import replace + +import pytest + +from openultrasast.config import PushConfig +from openultrasast.model.contracts import QuestionIdentity, RankedQuestion, ScopeDecision +from openultrasast.push.contracts import ComparisonAnalysis, PushComparison +from openultrasast.push.policy import ( + AdmissionCandidate, + AdmissionResult, + CandidateDelta, + CapabilityAdmission, + CapabilityKey, + EvidenceOperation, + admission_push_result, + admit_candidates, +) + + +def control(language="javascript", framework="express"): + q = QuestionIdentity("repository", language, "handler.src", "handler", "injection") + op = EvidenceOperation(q, "taint", "handler.src", 4, "exec(command)", "request.input", False, "{}") + delta = CandidateDelta( + "injection", + "handler.src:4:handler", + "model_entailed", + "request.input -> exec(command)", + "defect", + "new", + "source_connection_absent_from_comparable_base", + op, + (), + ("head:handler.src:2-2",), + "base", + "head", + "synthetic-semantics-v1", + ) + key = CapabilityKey(language, "server", framework, "injection", "taint", "shell-command", "synthetic-semantics-v1") + comparison = PushComparison("head", "base", "remote", ("refs/heads/main",)) + candidate = AdmissionCandidate(delta, key, comparison) + capability = CapabilityAdmission( + key, + "synthetic-only-evaluation", + "synthetic://reviewed-controls", + "PASS", + "Untrusted input reaching {operation} permits command execution in {context}.", + "At {operation}, use an argument-vector process API without a shell for {context}.", + enabled=True, + positive_controls=2, + reviewed_alerts=2, + calibration_artifact="synthetic://scorecard", + untouched_workload="synthetic-control-population", + operation_symbols=("exec",), + ) + scope = ScopeDecision("v1", "static", True, (RankedQuestion(q, 1.0, ()),), (), ()) + analysis = ComparisonAnalysis(comparison, (scope,), (q,), "complete_within_scope") + return candidate, capability, analysis + + +@pytest.mark.parametrize("language,framework", [("php", "wordpress"), ("javascript", "express")]) +def test_equivalent_origins_admit_same_claim_and_only_block_when_enabled(language, framework): + candidate, capability, analysis = control(language, framework) + admission = admit_candidates((candidate,), capabilities=(capability,)) + assert len(admission.defects) == 1 + defect = admission.defects[0] + assert defect.locations == (("handler.src", 4),) + assert defect.witnesses == (candidate.delta.witness,) + assert "exec(command)" in defect.repairs[0] and "shell-command" in defect.consequences[0] + for mode, disposition in (("advisory", "allow"), ("blocking", "block")): + result = admission_push_result(admission, (analysis,), config=PushConfig(mode=mode)) + assert result.finding_status == "actionable" + assert result.push_disposition == disposition + assert result.coverage_status == "complete_within_scope" + assert AdmissionResult.from_payload(admission.to_payload()) == admission + + +@pytest.mark.parametrize( + "mutation,reason", + [ + (lambda c: replace(c, delta=replace(c.delta, rung="suspicion")), "insufficient_rung"), + (lambda c: replace(c, delta=replace(c.delta, rung="model_corroborated")), "insufficient_rung"), + (lambda c: replace(c, delta=replace(c.delta, rung="execution_confirmed")), "insufficient_rung"), + (lambda c: replace(c, delta=replace(c.delta, novelty="unknown")), "comparison_unknown"), + (lambda c: replace(c, delta=replace(c.delta, novelty="unchanged")), "unchanged"), + (lambda c: replace(c, delta=replace(c.delta, witness=None)), "witness_unresolved"), + (lambda c: replace(c, delta=replace(c.delta, site="elsewhere:1:f")), "location_unresolved"), + (lambda c: replace(c, delta=replace(c.delta, head_operation=None)), "witness_unresolved"), + (lambda c: replace(c, delta=replace(c.delta, change_evidence=())), "change_unsupported"), + (lambda c: replace(c, dependency_gaps=("vendor-summary-missing",)), "dependency_unresolved"), + (lambda c: replace(c, sanitizer_contexts=("html-output",)), "sanitizer_semantics_mismatch"), + (lambda c: replace(c, delta=replace(c.delta, head_operation=replace(c.delta.head_operation, discharged=True))), "claim_discharged"), + (lambda c: replace(c, capability=replace(c.capability, language="python")), "capability_unavailable"), + ], +) +def test_rejected_claims_remain_diagnostics(mutation, reason): + candidate, capability, _ = control() + result = admit_candidates((mutation(candidate),), capabilities=(capability,)) + assert not result.defects + assert reason in result.dispositions[0].reasons + assert not result.dispositions[0].admitted + + +@pytest.mark.parametrize( + "mutation,reason", + [ + (lambda c: replace(c, enabled=False), "capability_disabled"), + (lambda c: replace(c, verdict="experimental"), "capability_unevaluated"), + (lambda c: replace(c, verdict="NO-GO"), "capability_unevaluated"), + (lambda c: replace(c, key=replace(c.key, semantics="stale")), "capability_unavailable"), + (lambda c: replace(c, key=replace(c.key, runtime="browser")), "capability_unavailable"), + (lambda c: replace(c, key=replace(c.key, framework="other")), "capability_unavailable"), + (lambda c: replace(c, repair_template="sanitize input"), "repair_unsupported"), + (lambda c: replace(c, consequence_template="potential security issue"), "consequence_unsupported"), + ], +) +def test_evaluated_semantics_and_grounded_repairs_required(mutation, reason): + candidate, capability, _ = control() + result = admit_candidates((candidate,), capabilities=(mutation(capability),)) + assert not result.defects + assert reason in result.dispositions[0].reasons + + +def test_no_default_eligibility_and_unsupported_origin_is_a_coverage_gap(): + candidate, _, analysis = control() + result = admit_candidates((candidate,)) + assert not result.defects and "capability_unavailable" in result.coverage_reasons + push = admission_push_result(result, (analysis,), config=PushConfig(mode="blocking")) + assert push.finding_status == "none" and push.coverage_status == "incomplete" and push.push_disposition == "allow" + strict = admission_push_result(result, (analysis,), config=PushConfig(mode="blocking", incomplete_coverage_policy="block")) + assert strict.push_disposition == "block" and strict.finding_status == "none" + advisory = admission_push_result(result, (analysis,), config=PushConfig(incomplete_coverage_policy="block")) + assert advisory.push_disposition == "allow" + + +def test_dedup_preserves_witnesses_and_comparisons_and_does_not_dismiss_printed_before(): + candidate, capability, _ = control() + second = replace( + candidate, + delta=replace(candidate.delta, witness="request.other -> exec(command)", base_revision="other-base"), + comparison=PushComparison("head", "other-base", "remote", ("refs/heads/release",)), + ) + result = admit_candidates((candidate, second, candidate), capabilities=(capability,)) + assert len(result.defects) == 1 and len(result.dispositions) == 3 + defect = result.defects[0] + assert len(defect.witnesses) == 2 and len(defect.comparisons) == 2 + assert defect.refs == ("refs/heads/main", "refs/heads/release") + assert admit_candidates((candidate, second, candidate), capabilities=(capability,)) == result + + +def test_unknown_novelty_is_coverage_but_unchanged_debt_is_not(): + candidate, capability, analysis = control() + for novelty, coverage in (("unknown", "incomplete"), ("unchanged", "complete_within_scope")): + admission = admit_candidates((replace(candidate, delta=replace(candidate.delta, novelty=novelty)),), capabilities=(capability,)) + assert admission_push_result(admission, (analysis,)).coverage_status == coverage + + +def test_ambiguous_capability_declarations_fail_closed(): + candidate, capability, _ = control() + result = admit_candidates((candidate,), capabilities=(capability, replace(capability, evaluation_id="other"))) + assert not result.defects and "capability_ambiguous" in result.coverage_reasons + + +def test_comparison_ownership_cannot_be_rebound_to_another_base_or_semantics(): + candidate, capability, _ = control() + for altered in ( + replace(candidate, comparison=replace(candidate.comparison, base_oid="different-base")), + replace(candidate, comparison=replace(candidate.comparison, head_oid="different-head")), + replace(candidate, delta=replace(candidate.delta, semantics="other-semantics")), + replace(candidate, delta=replace(candidate.delta, base_revision=None)), + ): + result = admit_candidates((altered,), capabilities=(capability,)) + assert not result.defects and "comparison_provenance_mismatch" in result.coverage_reasons + + +@pytest.mark.parametrize( + "field,value", [("positive_controls", 0), ("reviewed_alerts", 0), ("calibration_artifact", None), ("untouched_workload", None)] +) +def test_pass_label_without_evaluation_population_is_not_eligibility(field, value): + candidate, capability, _ = control() + result = admit_candidates((candidate,), capabilities=(replace(capability, **{field: value}),)) + assert not result.defects and "capability_unevaluated" in result.coverage_reasons + + +def test_unchanged_on_one_base_does_not_suppress_new_on_another(): + candidate, capability, analysis = control() + unchanged = replace( + candidate, + delta=replace(candidate.delta, base_revision="other-base", novelty="unchanged"), + comparison=PushComparison("head", "other-base", "remote", ("refs/heads/debt",)), + ) + result = admit_candidates((unchanged, candidate), capabilities=(capability,)) + assert len(result.defects) == 1 and result.defects[0].refs == candidate.comparison.refs + assert result.defects[0].comparisons == (candidate.comparison,) + with pytest.raises(ValueError, match="exact comparison"): + admission_push_result(result, (replace(analysis, comparison=unchanged.comparison),)) + + +def test_sql_operation_cannot_acquire_shell_consequence_from_same_family(): + candidate, capability, _ = control() + operation = replace(candidate.delta.head_operation, operation="query(sql)") + wrong_context = replace(candidate, delta=replace(candidate.delta, head_operation=operation, witness="req.query -> query(sql)")) + result = admit_candidates((wrong_context,), capabilities=(capability,)) + assert not result.defects + assert "operation_semantics_mismatch" in result.dispositions[0].reasons + assert result.dispositions[0].candidate.delta.witness == "req.query -> query(sql)" + + +@pytest.mark.parametrize("symbols", [(), ("execSync",), ("query",)]) +def test_missing_or_different_evaluated_operation_binding_is_diagnostic(symbols): + candidate, capability, _ = control() + result = admit_candidates((candidate,), capabilities=(replace(capability, operation_symbols=symbols),)) + assert not result.defects and "operation_semantics_mismatch" in result.coverage_reasons + + +@pytest.mark.parametrize("operation", ["query('exec(command)')", "custom.exec(command)", "execSync(command)"]) +def test_evaluated_operation_matching_preserves_receiver_and_symbol_boundaries(operation): + candidate, capability, _ = control() + candidate = replace( + candidate, delta=replace(candidate.delta, head_operation=replace(candidate.delta.head_operation, operation=operation)) + ) + assert not admit_candidates((candidate,), capabilities=(capability,)).defects