From 051f3b0058ac093ee21fbee74d8b09087ec435bc Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:04:20 +0200 Subject: [PATCH] feat(pre-push-safety-net): preserve uncertainty in benchmark scoring --- .kiro/specs/pre-push-safety-net/design.md | 6 + .kiro/specs/pre-push-safety-net/tasks.md | 3 +- .../measurements/2026-09-08-module-audit.json | 15 +- benchmarks/push/README.md | 45 +++ benchmarks/push/diagnostic-config-v1.json | 92 ++++++ benchmarks/push/diagnostic-profile-v1.json | 226 +++++++++++++ benchmarks/push/unmeasured-run-v1.json | 15 + .../2026-09-12-vampi-position-v2.json | 132 ++++++++ benchmarks/ranking/baselines/README.md | 20 ++ benchmarks/ranking/position.py | 111 +++++-- src/openultrasast/model/audit.py | 1 + src/openultrasast/push_scoring.py | 301 ++++++++++++++++++ tests/test_push_scoring.py | 262 +++++++++++++++ 13 files changed, 1197 insertions(+), 32 deletions(-) create mode 100644 benchmarks/push/diagnostic-config-v1.json create mode 100644 benchmarks/push/diagnostic-profile-v1.json create mode 100644 benchmarks/push/unmeasured-run-v1.json create mode 100644 benchmarks/ranking/baselines/2026-09-12-vampi-position-v2.json create mode 100644 benchmarks/ranking/baselines/README.md create mode 100644 src/openultrasast/push_scoring.py create mode 100644 tests/test_push_scoring.py diff --git a/.kiro/specs/pre-push-safety-net/design.md b/.kiro/specs/pre-push-safety-net/design.md index 27632da..e44905f 100644 --- a/.kiro/specs/pre-push-safety-net/design.md +++ b/.kiro/specs/pre-push-safety-net/design.md @@ -216,6 +216,12 @@ question/context/depth, facts/sanitizers, query code and relevant analysis confi also includes ranking/admission versions and any model/prompt identity. Comparison results include both base and head identities. Persist manifests with source byte counts and graph census. +These identities deliberately separate graph construction from ranking experiments. A ranking-only +change can reuse a compatible graph and unchanged query evidence while invalidating affected ranked +results. A query/fact change invalidates its answers without automatically rebuilding a graph whose +input/frontend/overlay semantics are unchanged. This supports iteration on the core as well as +repeated push checks; it does not authorize reusing stale evidence after a source change. + Only validated, complete artifacts can satisfy complete-result lookups. Partial artifacts may be retained for diagnostics but never promoted by cache presence. Publish atomically from private temporary paths under a per-key lock; another push may wait only within its deadline. Apply a diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index beec7ad..3d93336 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -38,12 +38,13 @@ started with group 1 (tasks 1.1–1.4); each task remains pending until independ - _Boundary: Evaluation harness_ - _Requirements: 6.1, 6.3, 8.3, 8.4_ - _Verified 2026-09-12:_ independent review APPROVED; 902 tests passed, 9 skipped, then 34 focused tests passed after a diagnostic-only correction. Fresh offline validation opened 506,502 bytes and retained all 11 cases/9 snapshots: 7 cases input-ready, 4 with named missing prerequisites. See `benchmarks/push/inputs.json` and `input-validation.json`. Ghost checkout/reviewed labels and independent per-capability PHP/Python populations remain missing; every capability stays experimental. No detector run or admission claim. -- [ ] 1.3 Make the baseline and push scorer fail honestly +- [x] 1.3 Make the baseline and push scorer fail honestly - Retain unmatched targets and unanswered queries in the declared population; require a queried witness before labeling a target transitively detected. - Emit actual vulnerable/fixed outcomes beside rank, selected mode, engine/facts/scope and elapsed stage costs. Freeze a versioned workload/profile before measuring. - Done when a deliberately missing target and failed query cannot improve reported recall, while a verified transitive positive is counted with its witness. - _Boundary: Evaluation harness, repository benchmark integration_ - _Requirements: 6.2, 6.3, 6.4, 6.5, 7.3_ + - _Verified 2026-09-12:_ independent review APPROVED; 925 tests passed, 9 skipped. Parent reproduced the frozen profile and actual VAmPI static rerun (15,942 pinned bytes, all 3 targets retained, SQLI unmatched, no transitive/detection claim, exit 1). Empty-run scoring retains 11 unresolved cases and explicitly reports unmeasured/0 recorded cases. Historical baselines remain unchanged with a caveat; new scorer consumes associated execution/review evidence and does not implement replay or admission. - [ ] 1.4 Establish typed push and engine integration contracts - Add importable push components and validated configuration for comparison base, shared deadline/cancellation allowance, local cache limit, advisory/blocking and separate coverage policy. - Define ownership of update/comparison, change context, selected/deferred scope, graph identity/lease and separate finding/coverage/push statuses as described in design. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 95e8285..c8ac2b0 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,9 +1,10 @@ { "counts": { "load_bearing": 92, - "standalone_capability": 8 + "standalone_capability": 9 }, - "modules": 100, + "modules": 101, + "note": "Inventory refreshed for pre-push-safety-net task 1.3 standalone scorer; original artifact filename retained for audit gate.", "orphaned": [], "rows": [ { @@ -610,6 +611,12 @@ "module": "push_inputs", "reason": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs" }, + { + "classification": "standalone_capability", + "importers": [], + "module": "push_scoring", + "reason": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring" + }, { "classification": "load_bearing", "importers": [ @@ -1076,10 +1083,10 @@ "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", "push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs", + "push_scoring": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring", "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", "slot_contract": "the slot/contract protocol stage_processors implements", "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" }, - "update_note": "Current tree inventory refreshed for pre-push-safety-net task 1.2 standalone input validator; filename retains the original audit date. No detection or ranking measurement.", - "updated_date": "2026-09-12" + "updated_at": "2026-09-12" } diff --git a/benchmarks/push/README.md b/benchmarks/push/README.md index d9a2f6e..2e546e2 100644 --- a/benchmarks/push/README.md +++ b/benchmarks/push/README.md @@ -44,3 +44,48 @@ Python populations are explicitly missing in `capability_prerequisites`. VAmPI r known targets (including SQLI), with `vulnerable=1`; these snapshot regressions are not paired pushes. Libpng is an envelope/unsupported-arithmetic control, not a detection success or a supported miss. The validator makes no finding, precision, recall, silence, latency or admission claim. + +## Frozen diagnostic scoring (task 1.3) + +`push_scoring` consumes recorded execution evidence; it does not run the scanner or +implement push delta/admission policy. Freeze the workload and expected configuration +**before** executing it: + +```sh +.venv/bin/python -m openultrasast.push_scoring freeze benchmarks/push/inputs.json benchmarks/push/diagnostic-config-v1.json > /tmp/profile.json +.venv/bin/python -m openultrasast.push_scoring score /tmp/profile.json benchmarks/push/input-validation.json benchmarks/push/unmeasured-run-v1.json +``` + +The committed v1 profile reserves the 11 declared cases and exact tip target locations. +It is a proposed 30-second evidence-mode experiment, not measured performance. The +`unmeasured-run-v1.json` deliberately contains no executions: scoring it exits **1**, +retains all 11 unresolved cases, and earns no recall, coverage or admission. An invalid +profile/runtime identity exits **2**. A fully resolved diagnostic score exits **0**; +that exit status is not capability admission. Freeze a new version when configuration, +source manifest, core, facts, query semantics or scope changes; do not edit a measured +profile to match later execution. Runtime `context` must record the actual values, +not merely copy an expected profile hash. Profiles are content identities, not signatures. + +Each `records` entry supplies `case_id`, actual `ranking_mode`, `rank` (or null), exact +`selected`/`deferred` question IDs, `queries`, `alerts`, `delta`, `coverage`, `cache_state` +and nonnegative elapsed `timings` including `total`. Cache states distinguish `cold`, +`warm_changed` and `identical_tip`; stage costs remain beside each outcome. Queries +supply `id`, `status` and `outcome`. A positive needs a finding with the exact frozen +`site` and `family`, a supported `rung`, and a nonempty actual `witness`. A transitive +claim additionally needs a recorded `origin` and ordered `flow` from that region to +the frozen target's path/function. Missing rank never proves transitivity. Negative +answers require the exact `target` and a witnessed discharge; empty/failed queries +cannot earn fixed-side silence. These checks establish evidence association, not the +truth of an arbitrary supplied witness: use engine output and review the attribution. + +Alerts are separate from raw findings. Only reviewed `actionable_true` alerts with +`review_evidence`, `question_id` and exact `site` association can contribute precision; +unreviewed/wrong-target alerts remain in its denominator. Benign comment changes retain +vulnerable backlog, so a raw positive is not automatically a new alert. The scorer +consumes a recorded delta; task 5.1 supplies replay, and the policy tasks establish it. + +Reports retain unresolved cases, sample counts, Wilson intervals, separate regression +results, per-capability metrics, coverage and cold/warm latency. Missing benign runs +have unknown interruption rate. VAmPI's readable regression may be measured while +independent admission prerequisites remain missing. Unsupported/unreviewed populations +stay visible. All results remain experimental pending the later joint admission gate. diff --git a/benchmarks/push/diagnostic-config-v1.json b/benchmarks/push/diagnostic-config-v1.json new file mode 100644 index 0000000..e8b56db --- /dev/null +++ b/benchmarks/push/diagnostic-config-v1.json @@ -0,0 +1,92 @@ +{ + "version": "pre-push-diagnostic-v1-2026-09-12", + "ranking_mode": "evidence", + "engine": "joern-4.0.625", + "facts": "642d643d5b40a20fe482ace566cc77d132870930ecd2f2e881dcecd7dc9a0641", + "queries": "64c1534edb07e770ab8631b48bc156ef54b71ddb8d8ee4357fd5022eb11f0229", + "policy": "diagnostic-only-no-admission-v1", + "scope": "first-party-only; declared vendor exclusion in graph and targets; exact selected/deferred question IDs required per record", + "hardware": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39", + "budget_seconds": 30, + "core": "7ac2bc93b35ce9d0cdcfc543266025519999ff45339657d4a5d283c67957d682", + "targets": { + "pmpro-introduce": { + "path": "classes/class.memberorder.php", + "function": "getMemberOrderByCode", + "family": "injection", + "lines": [ + 936 + ] + }, + "pmpro-repair": { + "path": "classes/class.memberorder.php", + "function": "getMemberOrderByCode", + "family": "injection", + "lines": [ + 937 + ] + }, + "pmpro-benign": { + "path": "classes/class.memberorder.php", + "function": "getMemberOrderByCode", + "family": "injection", + "lines": [ + 937 + ] + }, + "nodegoat-introduce": { + "path": "app/routes/contributions.js", + "function": "handleContributionsUpdate", + "family": "injection", + "lines": [ + 32, + 33, + 34 + ] + }, + "nodegoat-repair": { + "path": "app/routes/contributions.js", + "function": "handleContributionsUpdate", + "family": "injection", + "lines": [ + 32, + 33, + 34 + ] + }, + "nodegoat-benign": { + "path": "app/routes/contributions.js", + "function": "handleContributionsUpdate", + "family": "injection", + "lines": [ + 32, + 33, + 34 + ] + }, + "VAmPI-SQLI": { + "path": "models/user_model.py", + "function": "get_user", + "family": "injection", + "lines": [ + 73 + ] + }, + "VAmPI-BOLA": { + "path": "api_views/users.py", + "function": "update_password", + "family": "access_control", + "lines": [ + 187 + ] + }, + "VAmPI-BOLA-BOOKS": { + "path": "api_views/books.py", + "function": "get_by_title", + "family": "access_control", + "lines": [ + 51 + ] + } + } +} diff --git a/benchmarks/push/diagnostic-profile-v1.json b/benchmarks/push/diagnostic-profile-v1.json new file mode 100644 index 0000000..a61d61e --- /dev/null +++ b/benchmarks/push/diagnostic-profile-v1.json @@ -0,0 +1,226 @@ +{ + "schema_version": 1, + "manifest_sha256": "eec13cb19fe5e2c2402e61ebf4fd17fa809482f321e7a8e615215a12e1bcf845", + "cases": [ + { + "id": "pmpro-introduce", + "base": "pmpro-fixed", + "tip": "pmpro-vulnerable", + "label": "vulnerable", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.", + "prerequisites": [] + }, + { + "id": "pmpro-repair", + "base": "pmpro-vulnerable", + "tip": "pmpro-fixed", + "label": "fixed", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "PMPro upstream2.9.7 to2.9.8 parameter binding fix for CVE-2023-23488.", + "prerequisites": [] + }, + { + "id": "pmpro-benign", + "base": "pmpro-vulnerable", + "tip": "pmpro-benign", + "label": "benign", + "role": "development", + "capability": "php/wordpress/injection", + "provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.", + "prerequisites": [] + }, + { + "id": "nodegoat-introduce", + "base": "nodegoat-fixed", + "tip": "nodegoat-vulnerable", + "label": "vulnerable", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.", + "prerequisites": [] + }, + { + "id": "nodegoat-repair", + "base": "nodegoat-vulnerable", + "tip": "nodegoat-fixed", + "label": "fixed", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored replacement of three eval calls with Number conversion; source tutorial also documents alternate numeric parsing.", + "prerequisites": [] + }, + { + "id": "nodegoat-benign", + "base": "nodegoat-vulnerable", + "tip": "nodegoat-benign", + "label": "benign", + "role": "transfer", + "capability": "javascript/express/injection", + "provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.", + "prerequisites": [] + }, + { + "id": "VAmPI-SQLI", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/injection", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ] + }, + { + "id": "VAmPI-BOLA", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/access_control", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ] + }, + { + "id": "VAmPI-BOLA-BOOKS", + "base": "vampi-regression", + "tip": "vampi-regression", + "label": "regression", + "role": "regression", + "capability": "python/flask/access_control", + "provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.", + "prerequisites": [ + "Reviewed independent vulnerable/fixed/benign push pair required before hook admission." + ] + }, + { + "id": "libpng-CVE-2025-66293", + "base": "libpng-envelope", + "tip": "libpng-envelope", + "label": "unsupported", + "role": "envelope", + "capability": "c/libpng/memory_safety", + "provenance": "Arithmetic/bounds is outside the current abstract domain; measures graph/cost envelope only.", + "prerequisites": [] + }, + { + "id": "ghost-reserved-population", + "base": "ghost-reserved", + "tip": "ghost-reserved", + "label": "unreviewed", + "role": "reserved_holdout", + "capability": "javascript/express/unreviewed", + "provenance": "Reserved from production API/module/async shape and public security-fix history before scanner results. Manifest6.64.0-rc.0, Express4.22.2. Not an ESM or Express5 coverage claim.", + "prerequisites": [ + "Review and freeze supported positive/fixed/benign changes from independent history; select family before scanning.", + "Freeze core and existing facts after development; preserve reservation and record any contamination." + ] + } + ], + "config": { + "version": "pre-push-diagnostic-v1-2026-09-12", + "ranking_mode": "evidence", + "engine": "joern-4.0.625", + "facts": "642d643d5b40a20fe482ace566cc77d132870930ecd2f2e881dcecd7dc9a0641", + "queries": "64c1534edb07e770ab8631b48bc156ef54b71ddb8d8ee4357fd5022eb11f0229", + "policy": "diagnostic-only-no-admission-v1", + "scope": "first-party-only; declared vendor exclusion in graph and targets; exact selected/deferred question IDs required per record", + "hardware": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39", + "budget_seconds": 30, + "core": "7ac2bc93b35ce9d0cdcfc543266025519999ff45339657d4a5d283c67957d682", + "targets": { + "pmpro-introduce": { + "path": "classes/class.memberorder.php", + "function": "getMemberOrderByCode", + "family": "injection", + "lines": [ + 936 + ] + }, + "pmpro-repair": { + "path": "classes/class.memberorder.php", + "function": "getMemberOrderByCode", + "family": "injection", + "lines": [ + 937 + ] + }, + "pmpro-benign": { + "path": "classes/class.memberorder.php", + "function": "getMemberOrderByCode", + "family": "injection", + "lines": [ + 937 + ] + }, + "nodegoat-introduce": { + "path": "app/routes/contributions.js", + "function": "handleContributionsUpdate", + "family": "injection", + "lines": [ + 32, + 33, + 34 + ] + }, + "nodegoat-repair": { + "path": "app/routes/contributions.js", + "function": "handleContributionsUpdate", + "family": "injection", + "lines": [ + 32, + 33, + 34 + ] + }, + "nodegoat-benign": { + "path": "app/routes/contributions.js", + "function": "handleContributionsUpdate", + "family": "injection", + "lines": [ + 32, + 33, + 34 + ] + }, + "VAmPI-SQLI": { + "path": "models/user_model.py", + "function": "get_user", + "family": "injection", + "lines": [ + 73 + ] + }, + "VAmPI-BOLA": { + "path": "api_views/users.py", + "function": "update_password", + "family": "access_control", + "lines": [ + 187 + ] + }, + "VAmPI-BOLA-BOOKS": { + "path": "api_views/books.py", + "function": "get_by_title", + "family": "access_control", + "lines": [ + 51 + ] + } + } + }, + "gates": { + "precision": 0.95, + "recall": 0.9, + "completion": 0.95, + "warm_p95_seconds": 30.0, + "fixed_false_alerts": 0, + "benign_false_alerts": 0 + }, + "profile_sha256": "95c26cf474573766fa24c44f4375cfcb506b196412cd421deadff35b4eb9ef3f" +} diff --git a/benchmarks/push/unmeasured-run-v1.json b/benchmarks/push/unmeasured-run-v1.json new file mode 100644 index 0000000..50fed40 --- /dev/null +++ b/benchmarks/push/unmeasured-run-v1.json @@ -0,0 +1,15 @@ +{ + "profile_sha256": "95c26cf474573766fa24c44f4375cfcb506b196412cd421deadff35b4eb9ef3f", + "context": { + "ranking_mode": "evidence", + "engine": "joern-4.0.625", + "facts": "642d643d5b40a20fe482ace566cc77d132870930ecd2f2e881dcecd7dc9a0641", + "queries": "64c1534edb07e770ab8631b48bc156ef54b71ddb8d8ee4357fd5022eb11f0229", + "policy": "diagnostic-only-no-admission-v1", + "scope": "first-party-only; declared vendor exclusion in graph and targets; exact selected/deferred question IDs required per record", + "hardware": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39", + "budget_seconds": 30, + "core": "7ac2bc93b35ce9d0cdcfc543266025519999ff45339657d4a5d283c67957d682" + }, + "records": [] +} diff --git a/benchmarks/ranking/baselines/2026-09-12-vampi-position-v2.json b/benchmarks/ranking/baselines/2026-09-12-vampi-position-v2.json new file mode 100644 index 0000000..76e31fe --- /dev/null +++ b/benchmarks/ranking/baselines/2026-09-12-vampi-position-v2.json @@ -0,0 +1,132 @@ +[ + { + "schema_version": 2, + "measurement": "positional_coverage_not_detection", + "ranking_mode": "static", + "profile": { + "checkout": { + "commit": "f16052dce83f05847133ec98f01c5193a41de7d8", + "clean": true, + "status_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "version": "position-v2", + "requested_ranking_mode": "static", + "known_targets": [ + { + "id": "VAmPI-SQLI", + "family": "injection", + "file": "models/user_model.py", + "function": "get_user", + "line": 73, + "in_scope": true + }, + { + "id": "VAmPI-BOLA", + "family": "access_control", + "file": "api_views/users.py", + "function": "update_password", + "line": 187, + "in_scope": true + }, + { + "id": "VAmPI-BOLA-BOOKS", + "family": "access_control", + "file": "api_views/books.py", + "function": "get_by_title", + "line": 51, + "in_scope": true + } + ], + "input_witnesses": [ + { + "path": "api_views/books.py", + "bytes_read": 3043, + "sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8" + }, + { + "path": "api_views/users.py", + "bytes_read": 9257, + "sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c" + }, + { + "path": "models/user_model.py", + "bytes_read": 3642, + "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b" + } + ], + "implementation": { + "benchmarks/ranking/position.py": "c362e58843e669b68bb032574beff2342bd9597e0b451b38c727c1af643450e2", + "src/openultrasast/model/regions.py": "0ea1a7b153fad403848aa03f54ae45ea0dcf9c3d8b0f710238362de05139c84b", + "src/openultrasast/model/scan.py": "e11423755be4b72cfb6b5fd86b3d2fef68c0cc33aab997210f51430570256e93", + "src/openultrasast/model/evidence.py": "cda3619170a77bb8a4b7bb7fecb261e990699fe671714ccc830505a81029788d", + "src/openultrasast/model/layout.py": "dc9dc338dff5f1d4389d59d84ba4ddd249949470f9c2303df5ee70ad5d257f2d" + }, + "detection": "not_measured", + "sha256": "273dd99adcd94ec93df5e4c30018c145092141ba2c4e28a409d8bb5f8e6939a2" + }, + "detection_recall": null, + "supported_population": 3, + "unmatched_supported": 1, + "input_witnesses": [ + { + "path": "api_views/books.py", + "bytes_read": 3043, + "sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8" + }, + { + "path": "api_views/users.py", + "bytes_read": 9257, + "sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c" + }, + { + "path": "models/user_model.py", + "bytes_read": 3642, + "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b" + } + ], + "input_status": "readable", + "total_seconds": 0.793, + "repo": "vampi", + "regions": 25, + "distinct_ranks": 5, + "largest_tier_share": 0.36, + "largest_tier_rank": 1.0, + "budget_at_recall": null, + "budget_at_full_target_position_coverage": null, + "known": [ + { + "id": "VAmPI-SQLI", + "family": "injection", + "site": "models/user_model.py:73:get_user", + "position": null, + "rank": null, + "in_scope": true, + "reached_transitively": false, + "detection": "not_measured", + "position_status": "unresolved_target" + }, + { + "id": "VAmPI-BOLA", + "family": "access_control", + "site": "api_views/users.py:187:update_password", + "position": 15, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false, + "detection": "not_measured", + "position_status": "matched" + }, + { + "id": "VAmPI-BOLA-BOOKS", + "family": "access_control", + "site": "api_views/books.py:51:get_by_title", + "position": 10, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false, + "detection": "not_measured", + "position_status": "matched" + } + ] + } +] diff --git a/benchmarks/ranking/baselines/README.md b/benchmarks/ranking/baselines/README.md new file mode 100644 index 0000000..99e2503 --- /dev/null +++ b/benchmarks/ranking/baselines/README.md @@ -0,0 +1,20 @@ +# Historical ranking measurements + +The eight JSON artifacts dated 2026-09-10 and 2026-09-11 are preserved as recorded. +They predate the scoring correction in pre-push-safety-net task 1.3 and must not be +interpreted as end-to-end detection or pre-push acceptance results. + +An audit on 2026-09-12 read all eight artifacts. Each records VAmPI-SQLI with +`position: null`, `reached_transitively: true` and a repository `budget_at_recall` +of 16. The old position instrument assigned that transitive label solely because +it could not locate a matching region, then omitted the target from the budget +calculation. These artifacts contain no queried witness establishing that label. +This does not negate separately demonstrated VAmPI detections; it means this +particular metric cannot establish them. + +Other recorded positions, scores, and elapsed stage costs remain historical +observations under their recorded setup. Rank coverage is not detection recall, +and a stage duration is not total hook latency. Use the corrected instrument and +an explicit frozen profile for new comparisons. Retain unknown and unanswered +cases, and require a queried target witness before counting transitive detection. +Do not rewrite these files to make an earlier run appear to use the new semantics. diff --git a/benchmarks/ranking/position.py b/benchmarks/ranking/position.py index a01ed22..4f4100a 100644 --- a/benchmarks/ranking/position.py +++ b/benchmarks/ranking/position.py @@ -1,23 +1,19 @@ -"""Phase 0 of flow-aware-ranking: where does each pinned CVE sit in the order the scan actually examines? +"""Positional coverage baseline v2: ranks are not detection recall. -The ranker's success metric is not "is the CVE in the budget" -- on Paid Memberships Pro it already is, at -position 390 of 4,463 -- but **how small the budget can be while every CVE stays in it**. That number is -`budget_at_recall`, and until it is written down nothing later can claim to have improved it. - -This is the baseline. It runs no engine and asks no model: it is the region pipeline exactly as the scan -uses it, sorted exactly as the scan sorts it, with each known vulnerability's position read off. Offline, -against pinned checkouts, so two runs of one repository agree. - - ousast-venv/bin/python benchmarks/ranking/position.py # every pinned recipe with a checkout - ousast-venv/bin/python benchmarks/ranking/position.py --json # the same, as an artifact +Every declared target remains visible, including unmatched sites. No transitive detection +is inferred here: that requires recorded answered queries and target witnesses in the +push scorer. Historical v1 artifacts remain unchanged. """ from __future__ import annotations import argparse +import hashlib import json import os +import subprocess import sys +import time from collections import Counter from pathlib import Path @@ -101,10 +97,55 @@ def evidence_ordered_regions(root: Path, regions: list[ScanRegion], language: st def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidence: bool = False, language: str = "") -> dict[str, object]: - regions = ordered_regions(root) + started = time.monotonic() + inputs = [] + for path in sorted({str(item["file"]) for item in known}): + try: + content = (root / path).read_bytes() + if not content: + raise ValueError("empty source") + inputs.append(dict(path=path, bytes_read=len(content), sha256=hashlib.sha256(content).hexdigest())) + except (OSError, ValueError) as exc: + inputs.append(dict(path=path, bytes_read=0, problem=str(exc))) facts: dict[str, object] = {} + # Frozen identity recorded before the evidence pass; this is a rank-only experiment. + source_root = Path(__file__).resolve().parents[2] + version_paths = [ + "benchmarks/ranking/position.py", + "src/openultrasast/model/regions.py", + "src/openultrasast/model/scan.py", + "src/openultrasast/model/evidence.py", + "src/openultrasast/model/layout.py", + ] + checkout = {} + try: + commit = ( + subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, timeout=10).decode().strip() + ) + status = subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], stderr=subprocess.DEVNULL, timeout=10) + checkout = dict(commit=commit, clean=not status, status_sha256=hashlib.sha256(status).hexdigest()) + except (OSError, subprocess.SubprocessError): + checkout = dict(problem="checkout commit unavailable") + profile = dict( + checkout=checkout, + version="position-v2", + requested_ranking_mode="evidence" if by_evidence else "static", + known_targets=known, + input_witnesses=inputs, + implementation={path: hashlib.sha256((source_root / path).read_bytes()).hexdigest() for path in version_paths}, + detection="not_measured", + ) + profile["sha256"] = hashlib.sha256(json.dumps(profile, sort_keys=True).encode()).hexdigest() + regions = ordered_regions(root) if root.is_dir() else [] + actual_mode = "static" if by_evidence: - regions, facts = evidence_ordered_regions(root, regions, language) + try: + regions, facts = evidence_ordered_regions(root, regions, language) + actual_mode = "evidence" + except (OSError, RuntimeError, subprocess.SubprocessError, SystemExit) as exc: + regions = [] + actual_mode = "unavailable" + facts = dict(evidence_failure=str(exc)) tiers = Counter(r.rank for r in regions) largest = max(tiers.values()) if tiers else 0 rows: list[dict[str, object]] = [] @@ -112,7 +153,11 @@ def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidenc # A known vulnerability names a file and a function. Match on both, path by suffix because a recipe # writes it relative to the repository and a region may carry it the same way or deeper. hit = next( - (i for i, r in enumerate(regions) if r.path.endswith(str(item["file"])) and r.function == item["function"]), + ( + i + for i, r in enumerate(regions) + if r.path in (str(item["file"]), str(root / str(item["file"]))) and r.function == item["function"] + ), None, ) rows.append( @@ -120,26 +165,39 @@ def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidenc "id": item["id"], "family": item["family"], "site": f"{item['file']}:{item['line']}:{item['function']}", - # Zero-based. None means no region is named after this function: the site is a SINK that an - # entry-point region elsewhere reaches through `callDepth` -- VAmPI's SQL injection lives in - # `models/user_model.py:get_user` and is found from the handler that calls it. Its true - # position is that handler's, which only the flow can say, so it is reported and left out of - # `budget_at_recall` rather than counted as a miss it is not. + # Zero-based position is only a region match, never a completed detection. "position": hit, "rank": regions[hit].rank if hit is not None else None, "in_scope": item["in_scope"], - "reached_transitively": hit is None, + "reached_transitively": False, + "detection": "not_measured", + "position_status": "matched" if hit is not None else "unresolved_target", } ) positioned = [r["position"] for r in rows if r["position"] is not None and r["in_scope"]] + supported = [row for row in rows if row["in_scope"]] + all_located = bool(supported) and len(positioned) == len(supported) return { + "schema_version": 2, + "measurement": "positional_coverage_not_detection", + "ranking_mode": actual_mode, + "profile": profile, + "detection_recall": None, + "supported_population": len(supported), + "unmatched_supported": len(supported) - len(positioned), + "input_witnesses": inputs, + "input_status": "readable" if inputs and all(row["bytes_read"] for row in inputs) else "incomplete", + "total_seconds": round(time.monotonic() - started, 3), "repo": name, "regions": len(regions), "distinct_ranks": len(tiers), "largest_tier_share": round(largest / len(regions), 3) if regions else 0.0, "largest_tier_rank": max(tiers, key=lambda k: tiers[k]) if tiers else None, # The smallest examined-region budget that still holds every in-scope known vulnerability. - "budget_at_recall": (max(positioned) + 1) if positioned else None, + "budget_at_recall": None, # Retired misleading name: rank does not measure recall. + "budget_at_full_target_position_coverage": (max(positioned) + 1) + if all_located and all(row["bytes_read"] for row in inputs) + else None, "known": rows, **facts, } @@ -160,8 +218,7 @@ def main() -> int: continue root = checkout_path(recipe) if not root.is_dir(): - print(f"{recipe.name}: not checked out, skipping", file=sys.stderr) - continue + print(f"{recipe.name}: not checked out; retaining unresolved targets", file=sys.stderr) known = [ {"id": k.id, "family": k.family, "file": k.file, "function": k.function, "line": k.line, "in_scope": k.in_scope} for k in recipe.known @@ -173,20 +230,20 @@ def main() -> int: if args.json: print(json.dumps(results, indent=2)) - return 0 + return 1 if any(r["unmatched_supported"] or r["input_status"] != "readable" or r.get("evidence_failure") for r in results) else 0 for r in results: print( f"{r['repo']}: {r['regions']} regions, {r['distinct_ranks']} distinct ranks, " f"{100 * float(r['largest_tier_share']):.1f}% share rank {r['largest_tier_rank']}" ) for k in r["known"]: - where = "reached via another region" if k["position"] is None else f"position {k['position']} (rank {k['rank']})" + where = "unresolved target position" if k["position"] is None else f"position {k['position']} (rank {k['rank']})" scope = "" if k["in_scope"] else " [out of scope]" print(f" {k['id']:16} {where:28} {k['site']}{scope}") - print(f" budget_at_recall = {r['budget_at_recall']}") + print(f" budget_at_full_target_position_coverage = {r['budget_at_full_target_position_coverage']}; detection not measured") if "tier_counts" in r: print(f" build {r['build_seconds']}s, evidence {r['evidence_seconds']}s over {r['pairs']} pairs, tiers {r['tier_counts']}") - return 0 + return 1 if any(r["unmatched_supported"] or r["input_status"] != "readable" or r.get("evidence_failure") for r in results) else 0 if __name__ == "__main__": diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index 0c918a3..46ed0df 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -48,6 +48,7 @@ SPEC_OWNED = { } STANDALONE = { + "push_scoring": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring", "push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs", "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", diff --git a/src/openultrasast/push_scoring.py b/src/openultrasast/push_scoring.py new file mode 100644 index 0000000..49ba508 --- /dev/null +++ b/src/openultrasast/push_scoring.py @@ -0,0 +1,301 @@ +"""Versioned diagnostic scorer for recorded push experiments, independent of replay. + +Answers and reviewed alerts are supplied by the experiment; rank is never an answer. +This instrument checks provenance and exact witness associations, not witness semantics. +Capability admission remains the later independent evaluation gate. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +from typing import Any + +_CONTEXT = ("ranking_mode", "engine", "facts", "queries", "policy", "scope", "hardware", "budget_seconds", "core") + + +def _digest(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def _unique(rows: list[dict[str, Any]], key: str) -> dict[str, dict[str, Any]]: + indexed = {row[key]: row for row in rows} + if len(indexed) != len(rows): + raise ValueError(f"duplicate {key}") + return indexed + + +def freeze_profile(manifest: bytes, config: dict[str, Any]) -> dict[str, Any]: + """Freeze BEFORE executing queries; changing any comparison identity needs a new profile.""" + data = json.loads(manifest) + cases = data["cases"] + if data.get("schema_version") != 1 or not cases: + raise ValueError("unsupported or empty input manifest") + _unique(cases, "id") + for name in ("version", *_CONTEXT): + if name not in config or config[name] in (None, ""): + raise ValueError(f"profile requires {name}") + if config["ranking_mode"] not in ("static", "evidence"): + raise ValueError("unknown ranking mode") + if not math.isfinite(config["budget_seconds"]) or config["budget_seconds"] <= 0: + raise ValueError("budget must be finite and positive") + for case in cases: + if case["label"] not in ("unsupported", "unreviewed"): + target = config.get("targets", {}).get(case["id"], {}) + if not target.get("path") or not target.get("family") or not target.get("function") or not target.get("lines"): + raise ValueError(f"declare target before measuring {case['id']}") + if any(type(line) is not int or line < 1 for line in target["lines"]): + raise ValueError("target lines must be positive integers") + profile: dict[str, Any] = dict( + schema_version=1, + manifest_sha256=hashlib.sha256(manifest).hexdigest(), + cases=cases, + config=config, + gates=dict(precision=0.95, recall=0.90, completion=0.95, warm_p95_seconds=30.0, fixed_false_alerts=0, benign_false_alerts=0), + ) + # Own copies: mutating the caller's config cannot mutate a frozen profile. + profile = json.loads(json.dumps(profile, allow_nan=False)) + profile["profile_sha256"] = _digest(profile) + return profile + + +def _rate(numerator: int, denominator: int) -> dict[str, Any]: + """Observed rate and Wilson 95% interval; no samples is undefined, never perfect.""" + result: dict[str, Any] = dict(numerator=numerator, denominator=denominator, value=None, wilson95=None) + if denominator: + p, z = numerator / denominator, 1.959963984540054 + center = (p + z * z / (2 * denominator)) / (1 + z * z / denominator) + half = z * math.sqrt(p * (1 - p) / denominator + z * z / (4 * denominator**2)) / (1 + z * z / denominator) + result.update(value=p, wilson95=[max(0.0, center - half), min(1.0, center + half)]) + return result + + +def _witness(query: dict[str, Any], target: dict[str, Any]) -> dict[str, Any] | None: + finding = query.get("finding", {}) + sites = {f"{target['path']}:{line}" for line in target["lines"]} + if ( + query.get("status") == "answered" + and query.get("outcome") == "positive" + and finding.get("site") in sites + and finding.get("family") == target["family"] + and finding.get("rung") in ("model_entailed", "execution_confirmed") + and isinstance(finding.get("witness"), str) + and finding["witness"].strip() + ): + return dict(question_id=query["id"], **finding) + return None + + +def _case( + case: dict[str, Any], record: dict[str, Any] | None, profile: dict[str, Any], snapshots: dict[str, dict[str, Any]] +) -> dict[str, Any]: + row = dict(case, outcome="unresolved", reached_transitively=False, witnesses=[], complete=False, recorded=record) + if record is None: + row["problem"] = "no recorded execution" + return row + if case["label"] in ("unsupported", "unreviewed"): + row["problem"] = f"{case['label']} population; no detection claim" + return row + if any( + snapshots.get(case[key], {}).get("status") != "ready" or snapshots.get(case[key], {}).get("bytes_read", 0) <= 0 + for key in ("base", "tip") + ): + row["problem"] = "input bytes not verified" + return row + selected, deferred = record["selected"], record["deferred"] + if len(set(selected + deferred)) != len(selected + deferred): + raise ValueError("duplicate or overlapping selected/deferred questions") + queries = _unique(record["queries"], "id") + if not set(queries).issubset(selected): + row["problem"] = "query absent from selected scope" + return row + target = profile["config"]["targets"][case["id"]] + witnesses = [w for q in queries.values() if (w := _witness(q, target)) is not None] + complete = ( + bool(selected) + and not deferred + and set(queries) == set(selected) + and all(q.get("status") == "answered" and q.get("outcome") in ("positive", "negative") for q in queries.values()) + and record.get("coverage") == "complete" + ) + negative = any( + q.get("status") == "answered" and q.get("outcome") == "negative" and q.get("target") == target and isinstance(q.get("witness"), str) + and bool(q["witness"].strip()) + for q in queries.values() + ) + target_region = {key: target[key] for key in ("path", "function")} + transitive = any( + q.get("flow") + and len(q["flow"]) > 1 + and q["flow"][0] == q.get("origin") + and q["flow"][-1] == target_region + and q.get("origin") != target_region + and _witness(q, target) + for q in queries.values() + ) + row.update( + witnesses=witnesses, + complete=complete and bool(witnesses or negative), + outcome="positive" if witnesses else "negative" if complete and negative else "unresolved", + reached_transitively=bool(transitive), + ) + if row["outcome"] == "unresolved": + row["problem"] = "target unmatched, unanswered, or lacks exact witnessed answer" + return row + + +def _actionable(row: dict[str, Any], alert: dict[str, Any]) -> bool: + return ( + alert.get("review") == "actionable_true" + and bool(alert.get("review_evidence")) + and any(alert.get("question_id") == w["question_id"] and alert.get("site") == w["site"] for w in row["witnesses"]) + and row["label"] == "vulnerable" + and row["outcome"] == "positive" + and (row["recorded"] or {}).get("delta") in ("new", "worsened") + ) + + +def _metrics(rows: list[dict[str, Any]]) -> dict[str, Any]: + positive = [r for r in rows if r["label"] == "vulnerable"] + fixed = [r for r in rows if r["label"] == "fixed"] + benign = [r for r in rows if r["label"] == "benign"] + supported = [r for r in rows if r["label"] not in ("unsupported", "unreviewed")] + alerts = [(r, a) for r in rows for a in (r["recorded"] or {}).get("alerts", [])] + # Unreviewed emitted alerts remain in the precision denominator. A high rung alone is not actionability. + actionable = sum(_actionable(r, a) for r, a in alerts) + result = dict( + recall=_rate( + sum(r["outcome"] == "positive" and (r["recorded"] or {}).get("delta") in ("new", "worsened") for r in positive), len(positive) + ), + precision=_rate(actionable, len(alerts)), + fixed_side_silence=_rate(sum(r["outcome"] == "negative" and not r["recorded"].get("alerts") for r in fixed), len(fixed)), + benign_interruption=_rate(sum(bool((r["recorded"] or {}).get("alerts")) for r in benign), len(benign)), + completion=_rate(sum(r["complete"] for r in supported), len(supported)), + ) + latency = {} + for state in ("cold", "warm_changed", "identical_tip"): + values = sorted(r["recorded"]["timings"]["total"] for r in rows if r["recorded"] and r["recorded"].get("cache_state") == state) + latency[state] = dict( + samples=len(values), + p50=values[math.ceil(len(values) * 0.5) - 1] if values else None, + p95=values[math.ceil(len(values) * 0.95) - 1] if values else None, + ) + regression = [r for r in rows if r["label"] == "regression"] + result["regression_detection"] = _rate(sum(r["outcome"] == "positive" for r in regression), len(regression)) + unknown_benign = sum(r["recorded"] is None for r in benign) + result["benign_interruption"]["unobserved"] = unknown_benign + if unknown_benign: + result["benign_interruption"]["value"] = None + result["benign_interruption"]["wilson95"] = None + result["actionable_recall"] = _rate( + sum(any(_actionable(r, a) for a in (r["recorded"] or {}).get("alerts", [])) for r in positive), len(positive) + ) + declared_queries = sum(len(r["recorded"]["selected"]) + len(r["recorded"]["deferred"]) for r in supported if r["recorded"]) + answered_queries = sum( + q.get("status") == "answered" and q.get("outcome") in ("positive", "negative") and q.get("id") in r["recorded"]["selected"] + for r in supported + if r["recorded"] + for q in r["recorded"]["queries"] + ) + result["query_completion"] = _rate(answered_queries, declared_queries) + result["query_completion"]["unmeasured_cases"] = sum(r["recorded"] is None for r in supported) + if result["query_completion"]["unmeasured_cases"]: + result["query_completion"]["value"] = None + result["query_completion"]["wilson95"] = None + result["latency"] = latency + result["unresolved"] = sum(r["outcome"] == "unresolved" for r in rows) + result["timeouts"] = sum((r["recorded"] or {}).get("coverage") == "timeout" for r in rows) + return result + + +def score(profile: dict[str, Any], validation: dict[str, Any], run: dict[str, Any]) -> dict[str, Any]: + """Score a recorded experiment against its predeclared population, never a filtered result list.""" + identity = profile["profile_sha256"] + if _digest({k: v for k, v in profile.items() if k != "profile_sha256"}) != identity: + raise ValueError("mutated frozen profile") + if run["profile_sha256"] != identity or validation["manifest_sha256"] != profile["manifest_sha256"]: + raise ValueError("profile or manifest mismatch") + for key in _CONTEXT: + if run.get("context", {}).get(key) != profile["config"][key]: + raise ValueError(f"actual run context mismatch: {key}") + cases = _unique(profile["cases"], "id") + validated = _unique(validation["cases"], "id") + if set(cases) != set(validated) or any( + any(validated[key].get(field) != case.get(field) for field in case) for key, case in cases.items() + ): + raise ValueError("validation population or labels differ from frozen manifest") + records = _unique(run["records"], "case_id") + if not set(records).issubset(cases): + raise ValueError("undeclared result case") + for record in records.values(): + for field in ("rank", "selected", "deferred", "queries", "alerts", "delta", "coverage"): + if field not in record: + raise ValueError(f"record requires observed {field}") + if record["delta"] not in ("new", "worsened", "removed", "unchanged", "unknown"): + raise ValueError("unknown delta disposition") + if record["coverage"] not in ("complete", "incomplete", "timeout", "error"): + raise ValueError("unknown coverage status") + if record["ranking_mode"] != profile["config"]["ranking_mode"]: + raise ValueError("actual ranking mode differs from frozen profile") + times = record["timings"] + if not {"total", "build", "query"}.issubset(times) or any( + type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in times.values() + ): + raise ValueError("timings must contain finite nonnegative build, query and total elapsed seconds") + if record.get("cache_state") not in ("cold", "warm_changed", "identical_tip"): + raise ValueError("declare cold, warm_changed or identical_tip cache state") + snapshots = _unique(validation["snapshots"], "id") + rows = [_case(case, records.get(key), profile, snapshots) for key, case in cases.items()] + for row in rows: + row["input_status"] = validated[row["id"]]["status"] + row["input_problem"] = validated[row["id"]].get("problem") + if row["input_status"] == "invalid_input": + row.update(outcome="unresolved", complete=False, witnesses=[], reached_transitively=False, problem="invalid validated input") + return dict( + schema_version=1, + profile_sha256=identity, + profile=profile, + population=len(rows), + recorded_cases=len(records), + measurement_status="recorded" if records else "unmeasured", + cases=rows, + metrics=_metrics(rows), + by_capability={cap: _metrics([r for r in rows if r["capability"] == cap]) for cap in sorted({r["capability"] for r in rows})}, + admission="experimental", + gates_passed=False, + note=( + "Diagnostic instrument only. No admission without independent capability evaluation. " + "Missing runs contribute only conservative full-population bounds, not observed detector performance. " + "Witness association is checked; semantic/actionability review is supplied evidence, not inferred from rank." + ), + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + freeze = sub.add_parser("freeze") + freeze.add_argument("manifest", type=Path) + freeze.add_argument("config", type=Path) + measure = sub.add_parser("score") + measure.add_argument("profile", type=Path) + measure.add_argument("validation", type=Path) + measure.add_argument("run", type=Path) + args = parser.parse_args() + try: + if args.command == "freeze": + result = freeze_profile(args.manifest.read_bytes(), json.loads(args.config.read_text())) + else: + result = score(json.loads(args.profile.read_text()), json.loads(args.validation.read_text()), json.loads(args.run.read_text())) + except (ValueError, KeyError, TypeError, OSError) as exc: + print(json.dumps(dict(status="invalid_measurement", problem=str(exc)))) + return 2 + print(json.dumps(result, indent=2, allow_nan=False)) + return 1 if args.command == "score" and result["metrics"]["unresolved"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_push_scoring.py b/tests/test_push_scoring.py new file mode 100644 index 0000000..c916b41 --- /dev/null +++ b/tests/test_push_scoring.py @@ -0,0 +1,262 @@ +"""An absent answer must never improve a security score.""" + +import copy +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + +from openultrasast.push_scoring import freeze_profile, score + + +def inputs(): + cases = [ + dict(id=label, label=label, capability="javascript/express/injection", role="development", base="b", tip="h", prerequisites=[]) + for label in ("vulnerable", "fixed", "benign", "regression", "unreviewed") + ] + manifest = json.dumps(dict(schema_version=1, cases=cases)).encode() + config = dict( + version="test-v1", + core="core-test", + ranking_mode="static", + engine="joern-test", + facts="facts-test", + queries="queries-test", + policy="diagnostic-test", + scope="first-party-test", + hardware="test", + budget_seconds=30, + targets={case["id"]: dict(path="api.js", function="evalBody", lines=[4], family="injection") for case in cases}, + ) + profile = freeze_profile(manifest, config) + validation = dict( + manifest_sha256=hashlib.sha256(manifest).hexdigest(), + cases=[dict(case, status="ready") for case in cases], + snapshots=[dict(id="b", status="ready", bytes_read=42), dict(id="h", status="ready", bytes_read=42)], + ) + return profile, validation + + +def record(case="vulnerable", **updates): + value = dict( + case_id=case, + ranking_mode="static", + rank=None, + selected=["q-handler"], + deferred=[], + queries=[ + dict( + id="q-handler", + status="answered", + outcome="positive", + origin=dict(path="routes.js", function="handler"), + flow=[dict(path="routes.js", function="handler"), dict(path="api.js", function="evalBody")], + finding=dict( + site="api.js:4", family="injection", rung="model_entailed", witness="source req.body.value -> eval at api.js:4" + ), + ) + ], + alerts=[], + delta="new", + coverage="complete", + cache_state="cold", + timings=dict(build=1.0, query=2.0, total=3.0), + ) + value.update(updates) + return value + + +def run(records): + profile, validation = inputs() + return score(profile, validation, dict(profile_sha256=profile["profile_sha256"], context=profile["config"], records=records)) + + +def test_missing_and_failed_keep_positive_denominator(): + missing = run([]) + failed = run([record(queries=[dict(id="q-handler", status="failed", outcome="unknown")])]) + assert missing["metrics"]["recall"]["denominator"] == failed["metrics"]["recall"]["denominator"] == 1 + assert failed["metrics"]["recall"]["numerator"] == 0 + assert failed["cases"][0]["outcome"] == "unresolved" + assert len(missing["cases"]) == 5 + assert missing["recorded_cases"] == 0 + assert missing["measurement_status"] == "unmeasured" + + +def test_witnessed_positive_without_rank_is_counted(): + result = run([record()]) + assert result["metrics"]["recall"]["numerator"] == 1 + assert result["cases"][0]["reached_transitively"] is True + assert result["cases"][0]["witnesses"][0]["question_id"] == "q-handler" + + +@pytest.mark.parametrize("mutation", ["empty_witness", "wrong_site", "wrong_family", "unanswered", "unselected"]) +def test_positive_requires_answered_exact_target_witness(mutation): + row = record() + q = row["queries"][0] + if mutation == "empty_witness": + q["finding"]["witness"] = "" + elif mutation == "wrong_site": + q["finding"]["site"] = "other-api.js:4" + elif mutation == "wrong_family": + q["finding"]["family"] = "path" + elif mutation == "unanswered": + q["status"] = "failed" + else: + row["selected"] = [] + result = run([row]) + assert result["metrics"]["recall"]["numerator"] == 0 + assert result["cases"][0]["reached_transitively"] is False + + +def test_fixed_negative_needs_completed_answer_benign_backlog_is_not_alert(): + fixed = record( + "fixed", + queries=[ + dict( + id="q-handler", + status="answered", + outcome="negative", + target=dict(path="api.js", function="evalBody", lines=[4], family="injection"), + witness="bound parameter discharges target", + ) + ], + delta="removed", + ) + benign = record("benign", delta="unchanged") + result = run([fixed, benign]) + assert result["metrics"]["fixed_side_silence"]["numerator"] == 1 + assert result["metrics"]["benign_interruption"]["numerator"] == 0 + assert result["cases"][2]["outcome"] == "positive" + assert result["metrics"]["precision"]["denominator"] == 0 + failed = run([record("fixed", queries=[], coverage="incomplete")]) + assert failed["metrics"]["fixed_side_silence"]["numerator"] == 0 + + +def test_profiles_cannot_be_swapped_or_mutated(): + profile, validation = inputs() + for mutation in ("hash", "mode", "manifest", "profile"): + p, v = copy.deepcopy(profile), copy.deepcopy(validation) + data = dict(profile_sha256=p["profile_sha256"], context=profile["config"], records=[record()]) + if mutation == "hash": + data["profile_sha256"] = "other" + elif mutation == "mode": + data["records"][0]["ranking_mode"] = "evidence" + elif mutation == "manifest": + v["manifest_sha256"] = "other" + else: + p["config"]["facts"] = "new" + with pytest.raises(ValueError): + score(p, v, data) + + +def test_static_rank_never_claims_transitive_detection(tmp_path, monkeypatch): + path = Path(__file__).parents[1] / "benchmarks/ranking/position.py" + spec = importlib.util.spec_from_file_location("position", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "ordered_regions", lambda root: []) + (tmp_path / "api.js").write_text("eval(req.body.value)") + result = module.measure( + "test", tmp_path, [dict(id="missing", file="api.js", function="handler", family="injection", line=4, in_scope=True)] + ) + assert result["known"][0]["reached_transitively"] is False + assert result["supported_population"] == 1 + assert result["budget_at_recall"] is None + assert result["detection_recall"] is None + + +def test_rank_absence_without_path_is_not_transitivity(): + row = record() + del row["queries"][0]["flow"] + result = run([row]) + assert result["metrics"]["recall"]["numerator"] == 1 + assert result["cases"][0]["reached_transitively"] is False + + +@pytest.mark.parametrize("field", ["engine", "facts", "queries", "scope", "budget_seconds"]) +def test_actual_execution_context_must_match(field): + profile, validation = inputs() + context = dict(profile["config"], **{field: "different"}) + with pytest.raises(ValueError, match="context mismatch"): + score(profile, validation, dict(profile_sha256=profile["profile_sha256"], context=context, records=[])) + + +def test_unreviewed_wrong_target_alerts_remain_precision_denominator(): + row = record( + alerts=[ + dict(review="unreviewed"), + dict(review="actionable_true", review_evidence="review1", question_id="q-handler", site="other.js:4"), + ] + ) + result = run([row]) + assert result["metrics"]["precision"]["numerator"] == 0 + assert result["metrics"]["precision"]["denominator"] == 2 + assert result["gates_passed"] is False + + +def test_regression_readable_despite_missing_admission_prerequisite(): + profile, validation = inputs() + validation["cases"][3]["status"] = "missing_prerequisite" + validation["cases"][3]["problem"] = "independent holdout missing" + result = score( + profile, + validation, + dict(profile_sha256=profile["profile_sha256"], context=profile["config"], records=[record("regression", delta="unchanged")]), + ) + assert result["metrics"]["regression_detection"]["numerator"] == 1 + assert result["metrics"]["recall"]["numerator"] == 0 + assert result["admission"] == "experimental" + + +def test_missing_source_overrides_an_asserted_positive(): + profile, validation = inputs() + validation["snapshots"][1]["bytes_read"] = 0 + result = score(profile, validation, dict(profile_sha256=profile["profile_sha256"], context=profile["config"], records=[record()])) + assert result["metrics"]["recall"]["numerator"] == 0 + assert result["cases"][0]["outcome"] == "unresolved" + + +def test_unmeasured_benign_is_not_perfect_interruption_rate(): + assert run([])["metrics"]["benign_interruption"]["value"] is None + + +def test_evidence_launch_failure_retains_unresolved_population(tmp_path, monkeypatch): + path = Path(__file__).parents[1] / "benchmarks/ranking/position.py" + spec = importlib.util.spec_from_file_location("position_failure", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "ordered_regions", lambda root: []) + + def fail(*args): + raise SystemExit("no graph: input unreadable") + + monkeypatch.setattr(module, "evidence_ordered_regions", fail) + result = module.measure( + "test", + tmp_path, + [dict(id="missing", file="api.js", function="handler", family="injection", line=4, in_scope=True)], + by_evidence=True, + ) + assert result["ranking_mode"] == "unavailable" + assert result["supported_population"] == 1 + assert result["input_status"] == "incomplete" + assert "no graph" in result["evidence_failure"] + + +def test_deferred_questions_and_unmeasured_cases_remain_coverage_limits(): + result = run([record(deferred=["q-deferred"])]) + assert result["metrics"]["completion"]["numerator"] == 0 + assert result["metrics"]["query_completion"]["denominator"] == 2 + assert result["metrics"]["query_completion"]["numerator"] == 1 + assert result["metrics"]["query_completion"]["value"] is None + + +def test_actionable_recall_requires_reviewed_alert_beside_detection(): + result = run([record()]) + assert result["metrics"]["recall"]["numerator"] == 1 + assert result["metrics"]["actionable_recall"]["numerator"] == 0 + result = run([record(alerts=[dict(review="actionable_true", review_evidence="review1", question_id="q-handler", site="api.js:4")])]) + assert result["metrics"]["actionable_recall"]["numerator"] == 1 + assert result["metrics"]["precision"]["numerator"] == 1