diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index 4bfc1f7..0700fa3 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -440,6 +440,23 @@ Preserve all completed task evidence below. - _Depends: 12.3_ - _Requirements: 2.1, 2.2, 8.1_ + +- [ ] 13. Show the shared design transfers across languages (release milestone M3) +- [x] 13.1 Replay the declared Python family through the shared core + - Generalize the replay instrument so a case family is an argument rather than a hard-coded target: a harness that only knows one language cannot say whether the design transfers. Targets, pins and expectations come from the committed inputs. + - VAmPI is a regression snapshot with the same revision on both sides, so it can establish detection and completion but never novelty. The mode asserts that too: with nothing changed, a target reported new would be a comparison defect rather than a finding. + - _Verified 2026-09-17:_ all three declared targets detected at their declared lines with their questions answered, 35 of 35 questions complete on every case, nothing reported new, zero admitted defects. The SQL injection at `models/user_model.py:73` is the evidence the milestone names as previously unresolved. Two of the three are access control, the family that could not complete a question anywhere before group 12, so this is the first time broken object-level authorization has been asked on Python. Evidence: `benchmarks/measurements/2026-09-17-m3-python-transfer.json`. + - _Limits:_ identical revisions establish detection, not novelty; VAmPI is inspected teaching data and cannot qualify production Python support. + - _Boundary: Evaluation harness, full-run integration_ + - _Requirements: 6.2, 6.3, 8.1, 8.2_ +- [ ] 13.2 Replay the declared PHP pair through the shared core + - PMPro 2.9.7 against 2.9.8 with its upstream prepare fix, plus a comment-only benign variant. A different language, a different mechanism and a fix this project did not author. + - The first attempt selected one case of three, because the manifest labels a case by what it is while the case id says what the comparison does; the harness now maps label to role explicitly. That run also exhausted a 900-second budget on the single case it ran. + - Done when all three comparisons complete with their declared target evidence recorded, or the budget they need is stated as a finding rather than reported as a pass. + - _Boundary: Evaluation harness, full-run integration_ + - _Depends: 13.1_ + - _Requirements: 6.2, 6.3, 8.1, 8.2_ + ## Implementation Notes - Task5.2 measured PMPro timing out after both snapshots were materialized but before change context/graph analysis. Group6 must account for compatible snapshot/discovery reuse as well as graph reuse; identical-tip hits cannot substitute for the later representative changed-code latency gate. No new detector, scope heuristic or incremental CPG mutation is implied. diff --git a/benchmarks/measurements/2026-09-17-m3-python-transfer.json b/benchmarks/measurements/2026-09-17-m3-python-transfer.json new file mode 100644 index 0000000..a5c9fdd --- /dev/null +++ b/benchmarks/measurements/2026-09-17-m3-python-transfer.json @@ -0,0 +1,477 @@ +{ + "schema_version": 1, + "label": "EXPERIMENTAL recorded-veto evaluation (release milestone M1a). Listed vetoes were recorded, not applied. Nothing in this section is an alert, an admitted defect, a capability or a comparison result; the production result and admission above remain authoritative.", + "milestone": "M3", + "family": "VAmPI", + "language": "python", + "flag": "--experimental-record-vetoes", + "development_budget_seconds": 900.0, + "budget_note": "Explicitly named development budget; this run cannot satisfy or measure the hook latency gate.", + "expectations": { + "introduce": "raw finding at the declared target lines; recorded novelty new", + "repair": "fixed twin analyzed and quiet at the target; base-only findings retained", + "benign": "backlog unchanged at the target; never new or worsened", + "regression": "declared target still detected and its question answered; never new, because nothing changed" + }, + "pins": { + "vampi-regression": "f16052dce83f05847133ec98f01c5193a41de7d8" + }, + "targets": { + "VAmPI-SQLI": { + "path": "models/user_model.py", + "function": "get_user", + "family": "injection", + "lines": [ + 73 + ] + }, + "VAmPI-BOLA": { + "path": "api_views/users.py", + "function": "update_password", + "family": "access_control", + "lines": [ + 187 + ] + }, + "VAmPI-BOLA-BOOKS": { + "path": "api_views/books.py", + "function": "get_by_title", + "family": "access_control", + "lines": [ + 51 + ] + } + }, + "source_receipts": { + "vampi-regression": { + "path": "models/user_model.py", + "bytes": 3642, + "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b" + } + }, + "provenance": { + "repository": "/results/m3-python/objects.git", + "engine_runtime": "7213365694f836758cb5fe2b05504cbbb735b8c2f0dea5d5dd19546a1f5fd43d", + "engine": "joern-4.0.625", + "facts": "32edb6e1a78f1afb08ac4248cd436be16663e0d90dc1b3aaf948f2ce49a5a8f0", + "graph_layout": "403dda8d804c6109800ab95677f1c9a7c6158b9c83e58a65e2e6baa5132eefa8", + "queries": "deab634299983517c7e31903ed152f42345bb33e19841b4be86e3707a5372dcc", + "policy": "638f149e8d1e7e2f3a221f9a0b5cd5a70acee7954671f1fde94f4d30dd2b2b0b", + "core": "04ebc048a81554f16cdedcade0b055836bf44b6ded49e2491e12684bfc6d69fb", + "semantics": "4e2f7d24c98742124b4ea655c376e36c6af7fa0937d0e98fde81983b1e8dddb4", + "ranking_mode": "evidence", + "model": "disabled", + "capabilities": "9f0d85961ed49526a3131e71ec3f7e01aeca752da74ba7d74650168853781c9c", + "cache": "cold_no_reuse" + }, + "hardware": { + "platform": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39", + "cpu_affinity": [ + 0, + 1, + 2, + 3 + ] + }, + "instrument_sha256": "aa615f9184a997c764148b2f0742b69fd09e3c9142e15a1e2a493ff75203e179", + "cases": [ + { + "case_id": "VAmPI-SQLI", + "label": "regression", + "role": "regression", + "base_oid": "f16052dce83f05847133ec98f01c5193a41de7d8", + "head_oid": "f16052dce83f05847133ec98f01c5193a41de7d8", + "exit_code": 0, + "elapsed_seconds": 130.73618572601117, + "terminal": "No actionable defects reported; analysis is incomplete.\nNotice: Some checks are not enabled for normal alerts. Review capability coverage in the details.\nExperimental: recorded-veto evaluation retained 4 raw finding(s) with 16 recorded veto(es), 0 base-only finding(s) and 0 change-attributed evaluation finding(s) in the detailed result; recorded, not applied; not an alert.\nDetails: /results/m3-python/VAmPI-SQLI.json\n", + "provenance_matches": true, + "outcome": { + "expected": "declared target still detected and its question answered; never new, because nothing changed", + "identical_revisions": true, + "declared_target": { + "path": "models/user_model.py", + "function": "get_user", + "family": "injection", + "lines": [ + 73 + ] + }, + "findings_at_target": 1, + "finding_sites": [ + "models/user_model.py:73:get_user" + ], + "target_lines_hit": [ + 73 + ], + "target_questions": [ + { + "function": null, + "status": "completed", + "reason": "query_answered_and_arbitrated" + } + ], + "target_questions_answered": true, + "questions_completed": 35, + "questions_total": 35, + "base_questions_completed": 35, + "census": [ + { + "language": "python", + "files": 11 + } + ], + "production_result": [ + "none", + "incomplete" + ], + "recorded_status": "evaluated", + "recorded_at_target": [ + { + "site": "models/user_model.py:73:get_user", + "production": [ + "unknown", + "witness_identity_unresolved" + ], + "recorded": [ + "unknown", + "witness_identity_unresolved" + ], + "admission": [ + "capability_unavailable", + "comparison_unknown", + "witness_unresolved" + ], + "unlifted_vetoes": [ + [ + "comparison", + "witness_identity_unresolved" + ], + [ + "admission", + "capability_unavailable" + ], + [ + "admission", + "comparison_unknown" + ], + [ + "admission", + "witness_unresolved" + ], + [ + "admission", + "dependency_unresolved" + ] + ] + } + ], + "evaluation": [], + "base_only_findings": 0, + "coverage_reasons": [ + "capability_unavailable", + "comparison_unknown", + "witness_unresolved", + "eligibility:eligibility_provenance_mismatch", + "witness_identity_unresolved", + "dynamic_external_or_depth_context_unresolved:contributor-scan:5fd8c7c2cb3416fe25b4c1437ea95ffba902e60407139a6b5d5c9d0a898beb17", + "dynamic_external_or_depth_context_unresolved:contributor-scan:d3df6b543c4f932ffc44880f53a20d8e2a39b01aaed423c81c06fe8dff906acb", + "dynamic_external_or_depth_context_unresolved:contributor-scan:98d5d6114cd938cf0b0a384d35d279d5754c2aea55804b3dbb24a915753924b2", + "dynamic_external_or_depth_context_unresolved:contributor-scan:3aabf2177b0762d91115354177297f9b7fe6f3452c9c17ca2ff5a2cb03ecf98c", + "dynamic_external_or_depth_context_unresolved:contributor-scan:68865c15daec49375f4bfb2f0acf7e48f45fd56909f137b1574b5ebf09d01095", + "dynamic_external_or_depth_context_unresolved:contributor-scan:72b8e224c83e4e007aa6cab72f158df9b99f892db8994c65407e16ca4c929401", + "dynamic_external_or_depth_context_unresolved:contributor-scan:b61225a697d36ca5907ae4c3100b8f155ec5a24c9039353d0688d37797cb5806" + ], + "admitted_defects": 0, + "timings": { + "deadline_seconds": 900.0, + "cancellation_allowance_seconds": 2.0, + "resolution_seconds": 0.020111914025619626, + "provenance_seconds": 0.3533190260641277, + "snapshot_seconds": 0.10452717612497509, + "discovery_seconds": 0.8253818270750344, + "preparation_seconds": 0.9375425560865551, + "head_seconds": 110.84975008293986, + "base_seconds": 17.11559215700254, + "admission_seconds": 0.008097915910184383, + "recorded_vetoes_seconds": 0.769000818952918, + "cleanup_seconds": 0.0012579129543155432, + "total_seconds": 130.09027444501407 + } + } + }, + { + "case_id": "VAmPI-BOLA", + "label": "regression", + "role": "regression", + "base_oid": "f16052dce83f05847133ec98f01c5193a41de7d8", + "head_oid": "f16052dce83f05847133ec98f01c5193a41de7d8", + "exit_code": 0, + "elapsed_seconds": 6.416815093019977, + "terminal": "No actionable defects reported; analysis is incomplete.\nNotice: Some checks are not enabled for normal alerts. Review capability coverage in the details.\nExperimental: recorded-veto evaluation retained 4 raw finding(s) with 16 recorded veto(es), 0 base-only finding(s) and 0 change-attributed evaluation finding(s) in the detailed result; recorded, not applied; not an alert.\nDetails: /results/m3-python/VAmPI-BOLA.json\n", + "provenance_matches": true, + "outcome": { + "expected": "declared target still detected and its question answered; never new, because nothing changed", + "identical_revisions": true, + "declared_target": { + "path": "api_views/users.py", + "function": "update_password", + "family": "access_control", + "lines": [ + 187 + ] + }, + "findings_at_target": 1, + "finding_sites": [ + "api_views/users.py:187:update_password" + ], + "target_lines_hit": [ + 187 + ], + "target_questions": [ + { + "function": "delete_user", + "status": "completed", + "reason": "query_answered_and_arbitrated" + }, + { + "function": "me", + "status": "completed", + "reason": "query_answered_and_arbitrated" + }, + { + "function": "update_email", + "status": "completed", + "reason": "query_answered_and_arbitrated" + }, + { + "function": "update_password", + "status": "completed", + "reason": "query_answered_and_arbitrated" + } + ], + "target_questions_answered": true, + "questions_completed": 35, + "questions_total": 35, + "base_questions_completed": 35, + "census": [ + { + "language": "python", + "files": 11 + } + ], + "production_result": [ + "none", + "incomplete" + ], + "recorded_status": "evaluated", + "recorded_at_target": [ + { + "site": "api_views/users.py:187:update_password", + "production": [ + "unchanged", + "same_supported_mechanism" + ], + "recorded": [ + "unchanged", + "same_supported_mechanism" + ], + "admission": [ + "capability_unavailable", + "unchanged" + ], + "unlifted_vetoes": [ + [ + "admission", + "capability_unavailable" + ], + [ + "admission", + "unchanged" + ], + [ + "admission", + "dependency_unresolved" + ] + ] + } + ], + "evaluation": [], + "base_only_findings": 0, + "coverage_reasons": [ + "capability_unavailable", + "comparison_unknown", + "witness_unresolved", + "eligibility:eligibility_provenance_mismatch", + "witness_identity_unresolved", + "dynamic_external_or_depth_context_unresolved:contributor-scan:5fd8c7c2cb3416fe25b4c1437ea95ffba902e60407139a6b5d5c9d0a898beb17", + "dynamic_external_or_depth_context_unresolved:contributor-scan:d3df6b543c4f932ffc44880f53a20d8e2a39b01aaed423c81c06fe8dff906acb", + "dynamic_external_or_depth_context_unresolved:contributor-scan:98d5d6114cd938cf0b0a384d35d279d5754c2aea55804b3dbb24a915753924b2", + "dynamic_external_or_depth_context_unresolved:contributor-scan:3aabf2177b0762d91115354177297f9b7fe6f3452c9c17ca2ff5a2cb03ecf98c", + "dynamic_external_or_depth_context_unresolved:contributor-scan:68865c15daec49375f4bfb2f0acf7e48f45fd56909f137b1574b5ebf09d01095", + "dynamic_external_or_depth_context_unresolved:contributor-scan:72b8e224c83e4e007aa6cab72f158df9b99f892db8994c65407e16ca4c929401", + "dynamic_external_or_depth_context_unresolved:contributor-scan:b61225a697d36ca5907ae4c3100b8f155ec5a24c9039353d0688d37797cb5806" + ], + "admitted_defects": 0, + "timings": { + "deadline_seconds": 900.0, + "cancellation_allowance_seconds": 2.0, + "resolution_seconds": 0.01914159976877272, + "provenance_seconds": 0.294802316930145, + "snapshot_seconds": 0.09776705387048423, + "discovery_seconds": 0.01528664818033576, + "preparation_seconds": 0.120669448049739, + "head_seconds": 3.001164474990219, + "base_seconds": 1.547726898919791, + "admission_seconds": 0.00981250498443842, + "recorded_vetoes_seconds": 0.7769495439715683, + "cleanup_seconds": 0.0011470450554043055, + "total_seconds": 5.799134210916236 + } + } + }, + { + "case_id": "VAmPI-BOLA-BOOKS", + "label": "regression", + "role": "regression", + "base_oid": "f16052dce83f05847133ec98f01c5193a41de7d8", + "head_oid": "f16052dce83f05847133ec98f01c5193a41de7d8", + "exit_code": 0, + "elapsed_seconds": 6.3836759477853775, + "terminal": "No actionable defects reported; analysis is incomplete.\nNotice: Some checks are not enabled for normal alerts. Review capability coverage in the details.\nExperimental: recorded-veto evaluation retained 4 raw finding(s) with 16 recorded veto(es), 0 base-only finding(s) and 0 change-attributed evaluation finding(s) in the detailed result; recorded, not applied; not an alert.\nDetails: /results/m3-python/VAmPI-BOLA-BOOKS.json\n", + "provenance_matches": true, + "outcome": { + "expected": "declared target still detected and its question answered; never new, because nothing changed", + "identical_revisions": true, + "declared_target": { + "path": "api_views/books.py", + "function": "get_by_title", + "family": "access_control", + "lines": [ + 51 + ] + }, + "findings_at_target": 1, + "finding_sites": [ + "api_views/books.py:51:get_by_title" + ], + "target_lines_hit": [ + 51 + ], + "target_questions": [ + { + "function": "add_new_book", + "status": "completed", + "reason": "query_answered_and_arbitrated" + }, + { + "function": "get_by_title", + "status": "completed", + "reason": "query_answered_and_arbitrated" + } + ], + "target_questions_answered": true, + "questions_completed": 35, + "questions_total": 35, + "base_questions_completed": 35, + "census": [ + { + "language": "python", + "files": 11 + } + ], + "production_result": [ + "none", + "incomplete" + ], + "recorded_status": "evaluated", + "recorded_at_target": [ + { + "site": "api_views/books.py:51:get_by_title", + "production": [ + "unchanged", + "same_supported_mechanism" + ], + "recorded": [ + "unchanged", + "same_supported_mechanism" + ], + "admission": [ + "capability_unavailable", + "unchanged" + ], + "unlifted_vetoes": [ + [ + "admission", + "capability_unavailable" + ], + [ + "admission", + "unchanged" + ], + [ + "admission", + "dependency_unresolved" + ] + ] + } + ], + "evaluation": [], + "base_only_findings": 0, + "coverage_reasons": [ + "capability_unavailable", + "comparison_unknown", + "witness_unresolved", + "eligibility:eligibility_provenance_mismatch", + "witness_identity_unresolved", + "dynamic_external_or_depth_context_unresolved:contributor-scan:5fd8c7c2cb3416fe25b4c1437ea95ffba902e60407139a6b5d5c9d0a898beb17", + "dynamic_external_or_depth_context_unresolved:contributor-scan:d3df6b543c4f932ffc44880f53a20d8e2a39b01aaed423c81c06fe8dff906acb", + "dynamic_external_or_depth_context_unresolved:contributor-scan:98d5d6114cd938cf0b0a384d35d279d5754c2aea55804b3dbb24a915753924b2", + "dynamic_external_or_depth_context_unresolved:contributor-scan:3aabf2177b0762d91115354177297f9b7fe6f3452c9c17ca2ff5a2cb03ecf98c", + "dynamic_external_or_depth_context_unresolved:contributor-scan:68865c15daec49375f4bfb2f0acf7e48f45fd56909f137b1574b5ebf09d01095", + "dynamic_external_or_depth_context_unresolved:contributor-scan:72b8e224c83e4e007aa6cab72f158df9b99f892db8994c65407e16ca4c929401", + "dynamic_external_or_depth_context_unresolved:contributor-scan:b61225a697d36ca5907ae4c3100b8f155ec5a24c9039353d0688d37797cb5806" + ], + "admitted_defects": 0, + "timings": { + "deadline_seconds": 900.0, + "cancellation_allowance_seconds": 2.0, + "resolution_seconds": 0.017571233911439776, + "provenance_seconds": 0.28969555580988526, + "snapshot_seconds": 0.09667836595326662, + "discovery_seconds": 0.013358874013647437, + "preparation_seconds": 0.11729397182352841, + "head_seconds": 3.0182085398118943, + "base_seconds": 1.5452590121421963, + "admission_seconds": 0.010044088121503592, + "recorded_vetoes_seconds": 0.7472286818083376, + "cleanup_seconds": 0.0009872759692370892, + "total_seconds": 5.772516693919897 + } + } + } + ], + "verdict": { + "targets_detected": { + "VAmPI-SQLI": 1, + "VAmPI-BOLA": 1, + "VAmPI-BOLA-BOOKS": 1 + }, + "all_targets_detected": true, + "all_target_questions_answered": true, + "nothing_reported_new": true, + "all_recorded": true, + "no_admitted_defects": true, + "note": "Identical revisions on both sides: this establishes detection and completion, not novelty. A target reported new here would be a comparison defect." + }, + "bundle": { + "path": "2026-09-17-m3-python-transfer.json.gz", + "sha256": "b027bd653d33a2bcc760d02f57746f3a06cf76cd348b049a4c189e8dfcacfe20", + "files": 12 + }, + "finding": "All three declared Python targets are detected at their declared lines with their questions answered, and 35 of 35 questions complete on every case. Two of the three are access control, the family that could not complete a question anywhere before group 12, so this is the first time it has answered on Python. The SQL injection at models/user_model.py:73 is the VAmPI evidence the milestone spec names as previously unresolved.", + "limits": [ + "Identical revisions on both sides, so this establishes detection and completion, never novelty.", + "VAmPI is an inspected teaching application and cannot qualify production Python support.", + "No capability enabled, no admitted defect, no latency claim." + ] +} diff --git a/benchmarks/measurements/2026-09-17-m3-python-transfer.json.gz b/benchmarks/measurements/2026-09-17-m3-python-transfer.json.gz new file mode 100644 index 0000000..f17a0c0 Binary files /dev/null and b/benchmarks/measurements/2026-09-17-m3-python-transfer.json.gz differ diff --git a/benchmarks/push/transfer_replay.py b/benchmarks/push/transfer_replay.py new file mode 100644 index 0000000..424784a --- /dev/null +++ b/benchmarks/push/transfer_replay.py @@ -0,0 +1,309 @@ +"""M3: replay a declared vulnerable/fixed/benign family in any supported language. + +The NodeGoat instrument hard-codes its target and its three comparisons, which was right for M1 +and is wrong for a transfer claim: a harness that only knows one language cannot say whether the +design transfers. This one takes the case family as an argument and reads its target, pins and +expectations from the committed inputs, so PHP and Python are asked exactly the way Node was. + +What it records per revision is the same: the raw findings at the declared target, whether the +target question completed, the production comparison verdict, the recorded veto list, and the +census and source receipts. Expectations are written before the first scan. + +Development evidence under a named budget. It is not independent qualification, it enables no +capability, and it makes no latency claim. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from measure import git, pin_snapshot # noqa: E402 + +from openultrasast.model.scan import ScanBudget # noqa: E402 +from openultrasast.push.policy import CapabilityAdmission, CapabilityKey # noqa: E402 +from openultrasast.push.runner import _provenance # noqa: E402 +from openultrasast.push.vetoes import FLAG, LABEL # noqa: E402 +from openultrasast.push_inputs import validate_manifest # noqa: E402 +from openultrasast.repos import checkout_path, load_repo_recipe # noqa: E402 + +# What each declared comparison is expected to show, stated before anything runs. `introduce` +# reintroduces the reviewed operation, `repair` removes it, `benign` changes something else. +EXPECTATIONS = { + "introduce": "raw finding at the declared target lines; recorded novelty new", + "repair": "fixed twin analyzed and quiet at the target; base-only findings retained", + "benign": "backlog unchanged at the target; never new or worsened", + # A regression snapshot has the same revision on both sides, so there is no change and no + # novelty to claim. What it can show is whether the declared target is still detected and + # whether its question completes. Anything reported as new here would be a defect in the + # comparison, not a finding, so the verdict checks for exactly that. + "regression": "declared target still detected and its question answered; never new, because nothing changed", +} + + +# The manifest labels a case by what it IS (vulnerable, fixed, benign, regression); the case id +# says what the comparison DOES (introduce, repair, benign). Selecting on one and keying +# expectations by the other silently ran a single case and called the rest missing. +ROLES = {"vulnerable": "introduce", "fixed": "repair", "benign": "benign", "regression": "regression"} + + +def role_of(case: dict[str, Any]) -> str | None: + return ROLES.get(str(case.get("label", ""))) + + +def write(path: Path, data: object) -> None: + path.write_text(json.dumps(data, indent=2, allow_nan=False, default=str) + "\n") + + +def declaration_for(semantics: str, language: str, family: str, symbols: tuple[str, ...]) -> list[dict[str, Any]]: + """An unreviewed declaration so the evaluation explanation renders. Never enabled, never PASS.""" + key = CapabilityKey(language, "unspecified", "unspecified", family, "taint", "unreviewed", semantics) + declaration = CapabilityAdmission( + key, + "unreviewed-m3-transfer-v1", + "experimental://m3-transfer", + "experimental", + "A request value reaches {operation}, which interpolates it into a statement the runtime executes ({context} capability).", + "Bind the value at {operation} through the platform's parameterised form instead of interpolating it ({context} capability).", + operation_symbols=symbols, + ) + assert not declaration.enabled and declaration.verdict != "PASS", "the transfer declaration must stay unqualified" + return [declaration.to_payload()] + + +def summarize(case: dict[str, Any], target: dict[str, Any], artifact: dict[str, Any]) -> dict[str, Any]: + experimental = artifact.get("experimental") or {} + head = [record["scan"] for record in artifact["scans"] if record["side"] == "head"] + base = [record["scan"] for record in artifact["scans"] if record["side"] == "base"] + at_target = [ + finding + for scan in head + for finding in scan["findings"] + if finding["family"] == target["family"] and finding["site"].startswith(target["path"] + ":") + ] + questions = [ + outcome + for scan in head + for outcome in scan["question_outcomes"] + if outcome["identity"]["path"] == target["path"] and outcome["identity"]["family"] == target["family"] + ] + answered = bool(questions) and all(q["raw_rows_json"] is not None for q in questions) + findings = experimental.get("findings") if isinstance(experimental.get("findings"), list) else [] + recorded = [f for f in findings if f["family"] == target["family"] and f["site"].startswith(target["path"] + ":")] + return { + "expected": EXPECTATIONS[role_of(case) or "introduce"], + "identical_revisions": case["base"] == case["tip"], + "declared_target": target, + "findings_at_target": len(at_target), + "finding_sites": [f["site"] for f in at_target][:8], + "target_lines_hit": sorted({int(f["site"].split(":")[1]) for f in at_target if f["site"].split(":")[1].isdigit()}), + "target_questions": [{"function": q["identity"]["function"], "status": q["status"], "reason": q["reason"]} for q in questions], + "target_questions_answered": answered, + "questions_completed": sum(1 for s in head for q in s["question_outcomes"] if q["status"] == "completed"), + "questions_total": sum(len(s["question_outcomes"]) for s in head), + "base_questions_completed": sum(1 for s in base for q in s["question_outcomes"] if q["status"] == "completed"), + "census": [{"language": p.get("language"), "files": len(p.get("paths", ()))} for s in head for p in s.get("partitions", ())], + "production_result": [artifact["result"]["finding_status"], artifact["result"]["coverage_status"]], + "recorded_status": experimental.get("status"), + "recorded_at_target": [ + { + "site": f["site"], + "production": [f["production_novelty"], f["production_reason"]], + "recorded": [f["recorded_novelty"], f["recorded_reason"]], + "admission": f["production_admission"], + "unlifted_vetoes": [[v["stage"], v["veto"]] for v in f["vetoes"] if not v["lifted"]], + } + for f in recorded + ], + "evaluation": experimental.get("evaluation", []), + "base_only_findings": len(experimental.get("base_only_findings", []) or []), + "coverage_reasons": artifact["admission"]["coverage_reasons"][:12], + "admitted_defects": len(artifact["admission"]["defects"]), + "timings": {k: v for k, v in artifact["timings"].items() if k.endswith("_seconds")}, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--family", required=True, help="case id prefix, for example pmpro or nodegoat") + parser.add_argument("--language", required=True) + parser.add_argument("--symbols", default="", help="comma-separated operation symbols for the unreviewed declaration") + parser.add_argument("--deadline", type=float, default=900.0) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=False) + out = args.out.resolve() + + manifest = json.loads(args.manifest.read_bytes()) + targets = json.loads((args.manifest.parent / "diagnostic-config-v1.json").read_text())["targets"] + validation = validate_manifest(args.manifest, cache_root=args.cache, fetch=False) + write(out / "input-validation.json", validation) + ready = {s["id"] for s in validation["snapshots"] if s["status"] == "ready"} + prefix = args.family if args.family.endswith("-") else args.family + "-" + cases = [c for c in manifest["cases"] if c["id"].startswith(prefix) and role_of(c) is not None] + if not cases: + raise SystemExit(f"no vulnerable/fixed/benign cases for {args.family}") + snapshots = {s["id"]: s for s in manifest["snapshots"]} + missing = [c["id"] for c in cases if c["base"] not in ready or c["tip"] not in ready] + if missing: + raise SystemExit(f"inputs not ready: {missing}") + + recipe = load_repo_recipe(args.manifest.parent / snapshots[cases[0]["base"]]["recipe"]) + bare = out / "objects.git" + bare.mkdir() + git(bare, "init", "--bare") + alternates = set() + pins: dict[str, str] = {} + for case in cases: + for sid in (case["base"], case["tip"]): + if sid in pins: + continue + snapshot = snapshots[sid] + source = checkout_path(load_repo_recipe(args.manifest.parent / snapshot["recipe"]), args.cache) + objects = git(source, "rev-parse", "--path-format=absolute", "--git-path", "objects").decode().strip() + alternates.add(objects) + (bare / "objects/info/alternates").write_text("\n".join(sorted(alternates)) + "\n") + pins[sid] = pin_snapshot(bare, snapshot.get("commit", recipe.commit), snapshot) + + settings = ScanBudget(max_model_calls=0, max_regions=500, order_by_evidence=True) + installed = _provenance(bare, settings) + family = next((targets[c["id"]]["family"] for c in cases if c["id"] in targets), "injection") + symbols = tuple(s.strip() for s in args.symbols.split(",") if s.strip()) + declarations = out / "experimental-declarations.json" + write(declarations, declaration_for(installed["semantics"], args.language, family, symbols)) + + record: dict[str, Any] = { + "schema_version": 1, + "label": LABEL, + "milestone": "M3", + "family": args.family, + "language": args.language, + "flag": FLAG, + "development_budget_seconds": args.deadline, + "budget_note": "Explicitly named development budget; this run cannot satisfy or measure the hook latency gate.", + "expectations": EXPECTATIONS, + "pins": pins, + "targets": {c["id"]: targets.get(c["id"]) for c in cases}, + "source_receipts": { + sid: { + "path": targets[cases[0]["id"]]["path"], + "bytes": len(git(bare, "cat-file", "blob", f"{oid}:{targets[cases[0]['id']]['path']}")), + "sha256": hashlib.sha256(git(bare, "cat-file", "blob", f"{oid}:{targets[cases[0]['id']]['path']}")).hexdigest(), + } + for sid, oid in pins.items() + }, + "provenance": installed, + "hardware": {"platform": platform.platform(), "cpu_affinity": sorted(os.sched_getaffinity(0))}, + "instrument_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "cases": [], + } + write(out / "transfer-record.json", record) + + for case in cases: + target = targets.get(case["id"]) + if target is None: + record["cases"].append({"case_id": case["id"], "status": "no_declared_target"}) + write(out / "transfer-record.json", record) + continue + artifact = out / (case["id"] + ".json") + command = [ + sys.executable, + "-m", + "openultrasast.cli", + "pre-push", + str(bare), + "--base", + pins[case["base"]], + "--head", + pins[case["tip"]], + "--artifact", + str(artifact), + "--deadline", + str(args.deadline), + "--cancellation-allowance", + "2", + "--max-regions", + "500", + "--cache-dir", + str(out / "cache"), + FLAG, + "--experimental-declarations", + str(declarations), + ] + started = time.monotonic() + process = subprocess.run(command, capture_output=True, text=True, timeout=args.deadline + 60) # noqa: S603 + elapsed = time.monotonic() - started + (out / (case["id"] + ".stdout")).write_text(process.stdout) + (out / (case["id"] + ".stderr")).write_text(process.stderr) + entry: dict[str, Any] = { + "case_id": case["id"], + "label": case["label"], + "role": role_of(case), + "base_oid": pins[case["base"]], + "head_oid": pins[case["tip"]], + "exit_code": process.returncode, + "elapsed_seconds": elapsed, + "terminal": process.stdout, + } + if artifact.is_file(): + data = json.loads(artifact.read_text()) + entry["provenance_matches"] = all( + data["provenance"].get(k) == installed[k] for k in ("engine", "facts", "queries", "policy", "core", "semantics") + ) + entry["outcome"] = summarize(case, target, data) + else: + entry["outcome"] = {"instrument_error": "missing_artifact"} + record["cases"].append(entry) + write(out / "transfer-record.json", record) + print(json.dumps({"case": case["id"], "elapsed": round(elapsed, 1), "exit": process.returncode}), flush=True) + + outcomes = {c["case_id"]: c.get("outcome", {}) for c in record["cases"]} + regressions = [(c["case_id"], c.get("outcome", {})) for c in record["cases"] if c.get("role") == "regression"] + if regressions: + record["verdict"] = { + "targets_detected": {cid: o.get("findings_at_target", 0) for cid, o in regressions}, + "all_targets_detected": all(o.get("findings_at_target") for _, o in regressions), + "all_target_questions_answered": all(o.get("target_questions_answered") for _, o in regressions), + "nothing_reported_new": not any( + f["recorded"][0] in ("new", "worsened") for _, o in regressions for f in o.get("recorded_at_target", []) + ), + "all_recorded": all(o.get("recorded_status") == "evaluated" for _, o in regressions), + "no_admitted_defects": all(o.get("admitted_defects") == 0 for _, o in regressions), + "note": ( + "Identical revisions on both sides: this establishes detection and completion, not novelty. " + "A target reported new here would be a comparison defect." + ), + } + write(out / "transfer-record.json", record) + print(json.dumps(record["verdict"]), flush=True) + return 0 if all(v for k, v in record["verdict"].items() if k not in ("note", "targets_detected")) else 1 + by_role = {c.get("role"): c.get("outcome", {}) for c in record["cases"]} + introduce, repair, benign = by_role.get("introduce", {}), by_role.get("repair", {}), by_role.get("benign", {}) + record["verdict"] = { + "raw_witness_present": bool(introduce.get("findings_at_target")), + "fixed_twin_quiet": bool(repair.get("target_questions_answered")) and repair.get("findings_at_target") == 0, + "unchanged_not_new_regression": bool(benign.get("target_questions_answered")) + and not any(f["recorded"][0] in ("new", "worsened") for f in benign.get("recorded_at_target", [])), + "all_recorded": all(o.get("recorded_status") == "evaluated" for o in outcomes.values() if o), + "no_admitted_defects": all(o.get("admitted_defects") == 0 for o in outcomes.values() if o), + "note": "Development transfer evidence. Silence from a failure is not a pass; an unanalyzed fixed twin fails.", + } + write(out / "transfer-record.json", record) + print(json.dumps(record["verdict"]), flush=True) + return 0 if all(v for k, v in record["verdict"].items() if k != "note") else 1 + + +if __name__ == "__main__": + raise SystemExit(main())