mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(pre-push-safety-net): preserve uncertainty in benchmark scoring
This commit is contained in:
Marius Ciepluch
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"counts": {
|
||||
"load_bearing": 92,
|
||||
"standalone_capability": 8
|
||||
"standalone_capability": 9
|
||||
},
|
||||
"modules": 100,
|
||||
"modules": 101,
|
||||
"note": "Inventory refreshed for pre-push-safety-net task 1.3 standalone scorer; original artifact filename retained for audit gate.",
|
||||
"orphaned": [],
|
||||
"rows": [
|
||||
{
|
||||
@@ -610,6 +611,12 @@
|
||||
"module": "push_inputs",
|
||||
"reason": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs"
|
||||
},
|
||||
{
|
||||
"classification": "standalone_capability",
|
||||
"importers": [],
|
||||
"module": "push_scoring",
|
||||
"reason": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring"
|
||||
},
|
||||
{
|
||||
"classification": "load_bearing",
|
||||
"importers": [
|
||||
@@ -1076,10 +1083,10 @@
|
||||
"hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp",
|
||||
"mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration",
|
||||
"push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs",
|
||||
"push_scoring": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring",
|
||||
"skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path",
|
||||
"slot_contract": "the slot/contract protocol stage_processors implements",
|
||||
"stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate"
|
||||
},
|
||||
"update_note": "Current tree inventory refreshed for pre-push-safety-net task 1.2 standalone input validator; filename retains the original audit date. No detection or ranking measurement.",
|
||||
"updated_date": "2026-09-12"
|
||||
"updated_at": "2026-09-12"
|
||||
}
|
||||
|
||||
@@ -44,3 +44,48 @@ Python populations are explicitly missing in `capability_prerequisites`. VAmPI r
|
||||
known targets (including SQLI), with `vulnerable=1`; these snapshot regressions are not paired pushes.
|
||||
Libpng is an envelope/unsupported-arithmetic control, not a detection success or a supported miss.
|
||||
The validator makes no finding, precision, recall, silence, latency or admission claim.
|
||||
|
||||
## Frozen diagnostic scoring (task 1.3)
|
||||
|
||||
`push_scoring` consumes recorded execution evidence; it does not run the scanner or
|
||||
implement push delta/admission policy. Freeze the workload and expected configuration
|
||||
**before** executing it:
|
||||
|
||||
```sh
|
||||
.venv/bin/python -m openultrasast.push_scoring freeze benchmarks/push/inputs.json benchmarks/push/diagnostic-config-v1.json > /tmp/profile.json
|
||||
.venv/bin/python -m openultrasast.push_scoring score /tmp/profile.json benchmarks/push/input-validation.json benchmarks/push/unmeasured-run-v1.json
|
||||
```
|
||||
|
||||
The committed v1 profile reserves the 11 declared cases and exact tip target locations.
|
||||
It is a proposed 30-second evidence-mode experiment, not measured performance. The
|
||||
`unmeasured-run-v1.json` deliberately contains no executions: scoring it exits **1**,
|
||||
retains all 11 unresolved cases, and earns no recall, coverage or admission. An invalid
|
||||
profile/runtime identity exits **2**. A fully resolved diagnostic score exits **0**;
|
||||
that exit status is not capability admission. Freeze a new version when configuration,
|
||||
source manifest, core, facts, query semantics or scope changes; do not edit a measured
|
||||
profile to match later execution. Runtime `context` must record the actual values,
|
||||
not merely copy an expected profile hash. Profiles are content identities, not signatures.
|
||||
|
||||
Each `records` entry supplies `case_id`, actual `ranking_mode`, `rank` (or null), exact
|
||||
`selected`/`deferred` question IDs, `queries`, `alerts`, `delta`, `coverage`, `cache_state`
|
||||
and nonnegative elapsed `timings` including `total`. Cache states distinguish `cold`,
|
||||
`warm_changed` and `identical_tip`; stage costs remain beside each outcome. Queries
|
||||
supply `id`, `status` and `outcome`. A positive needs a finding with the exact frozen
|
||||
`site` and `family`, a supported `rung`, and a nonempty actual `witness`. A transitive
|
||||
claim additionally needs a recorded `origin` and ordered `flow` from that region to
|
||||
the frozen target's path/function. Missing rank never proves transitivity. Negative
|
||||
answers require the exact `target` and a witnessed discharge; empty/failed queries
|
||||
cannot earn fixed-side silence. These checks establish evidence association, not the
|
||||
truth of an arbitrary supplied witness: use engine output and review the attribution.
|
||||
|
||||
Alerts are separate from raw findings. Only reviewed `actionable_true` alerts with
|
||||
`review_evidence`, `question_id` and exact `site` association can contribute precision;
|
||||
unreviewed/wrong-target alerts remain in its denominator. Benign comment changes retain
|
||||
vulnerable backlog, so a raw positive is not automatically a new alert. The scorer
|
||||
consumes a recorded delta; task 5.1 supplies replay, and the policy tasks establish it.
|
||||
|
||||
Reports retain unresolved cases, sample counts, Wilson intervals, separate regression
|
||||
results, per-capability metrics, coverage and cold/warm latency. Missing benign runs
|
||||
have unknown interruption rate. VAmPI's readable regression may be measured while
|
||||
independent admission prerequisites remain missing. Unsupported/unreviewed populations
|
||||
stay visible. All results remain experimental pending the later joint admission gate.
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"version": "pre-push-diagnostic-v1-2026-09-12",
|
||||
"ranking_mode": "evidence",
|
||||
"engine": "joern-4.0.625",
|
||||
"facts": "642d643d5b40a20fe482ace566cc77d132870930ecd2f2e881dcecd7dc9a0641",
|
||||
"queries": "64c1534edb07e770ab8631b48bc156ef54b71ddb8d8ee4357fd5022eb11f0229",
|
||||
"policy": "diagnostic-only-no-admission-v1",
|
||||
"scope": "first-party-only; declared vendor exclusion in graph and targets; exact selected/deferred question IDs required per record",
|
||||
"hardware": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39",
|
||||
"budget_seconds": 30,
|
||||
"core": "7ac2bc93b35ce9d0cdcfc543266025519999ff45339657d4a5d283c67957d682",
|
||||
"targets": {
|
||||
"pmpro-introduce": {
|
||||
"path": "classes/class.memberorder.php",
|
||||
"function": "getMemberOrderByCode",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
936
|
||||
]
|
||||
},
|
||||
"pmpro-repair": {
|
||||
"path": "classes/class.memberorder.php",
|
||||
"function": "getMemberOrderByCode",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
937
|
||||
]
|
||||
},
|
||||
"pmpro-benign": {
|
||||
"path": "classes/class.memberorder.php",
|
||||
"function": "getMemberOrderByCode",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
937
|
||||
]
|
||||
},
|
||||
"nodegoat-introduce": {
|
||||
"path": "app/routes/contributions.js",
|
||||
"function": "handleContributionsUpdate",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
32,
|
||||
33,
|
||||
34
|
||||
]
|
||||
},
|
||||
"nodegoat-repair": {
|
||||
"path": "app/routes/contributions.js",
|
||||
"function": "handleContributionsUpdate",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
32,
|
||||
33,
|
||||
34
|
||||
]
|
||||
},
|
||||
"nodegoat-benign": {
|
||||
"path": "app/routes/contributions.js",
|
||||
"function": "handleContributionsUpdate",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
32,
|
||||
33,
|
||||
34
|
||||
]
|
||||
},
|
||||
"VAmPI-SQLI": {
|
||||
"path": "models/user_model.py",
|
||||
"function": "get_user",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
73
|
||||
]
|
||||
},
|
||||
"VAmPI-BOLA": {
|
||||
"path": "api_views/users.py",
|
||||
"function": "update_password",
|
||||
"family": "access_control",
|
||||
"lines": [
|
||||
187
|
||||
]
|
||||
},
|
||||
"VAmPI-BOLA-BOOKS": {
|
||||
"path": "api_views/books.py",
|
||||
"function": "get_by_title",
|
||||
"family": "access_control",
|
||||
"lines": [
|
||||
51
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"manifest_sha256": "eec13cb19fe5e2c2402e61ebf4fd17fa809482f321e7a8e615215a12e1bcf845",
|
||||
"cases": [
|
||||
{
|
||||
"id": "pmpro-introduce",
|
||||
"base": "pmpro-fixed",
|
||||
"tip": "pmpro-vulnerable",
|
||||
"label": "vulnerable",
|
||||
"role": "development",
|
||||
"capability": "php/wordpress/injection",
|
||||
"provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "pmpro-repair",
|
||||
"base": "pmpro-vulnerable",
|
||||
"tip": "pmpro-fixed",
|
||||
"label": "fixed",
|
||||
"role": "development",
|
||||
"capability": "php/wordpress/injection",
|
||||
"provenance": "PMPro upstream2.9.7 to2.9.8 parameter binding fix for CVE-2023-23488.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "pmpro-benign",
|
||||
"base": "pmpro-vulnerable",
|
||||
"tip": "pmpro-benign",
|
||||
"label": "benign",
|
||||
"role": "development",
|
||||
"capability": "php/wordpress/injection",
|
||||
"provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "nodegoat-introduce",
|
||||
"base": "nodegoat-fixed",
|
||||
"tip": "nodegoat-vulnerable",
|
||||
"label": "vulnerable",
|
||||
"role": "transfer",
|
||||
"capability": "javascript/express/injection",
|
||||
"provenance": "Authored replay direction reintroducing the reviewed operation; not a claim about the historical introducing commit.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "nodegoat-repair",
|
||||
"base": "nodegoat-vulnerable",
|
||||
"tip": "nodegoat-fixed",
|
||||
"label": "fixed",
|
||||
"role": "transfer",
|
||||
"capability": "javascript/express/injection",
|
||||
"provenance": "Authored replacement of three eval calls with Number conversion; source tutorial also documents alternate numeric parsing.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "nodegoat-benign",
|
||||
"base": "nodegoat-vulnerable",
|
||||
"tip": "nodegoat-benign",
|
||||
"label": "benign",
|
||||
"role": "transfer",
|
||||
"capability": "javascript/express/injection",
|
||||
"provenance": "Authored comment-only change: no new defect; vulnerable backlog remains. This label does not mean repository-clean.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-SQLI",
|
||||
"base": "vampi-regression",
|
||||
"tip": "vampi-regression",
|
||||
"label": "regression",
|
||||
"role": "regression",
|
||||
"capability": "python/flask/injection",
|
||||
"provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.",
|
||||
"prerequisites": [
|
||||
"Reviewed independent vulnerable/fixed/benign push pair required before hook admission."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA",
|
||||
"base": "vampi-regression",
|
||||
"tip": "vampi-regression",
|
||||
"label": "regression",
|
||||
"role": "regression",
|
||||
"capability": "python/flask/access_control",
|
||||
"provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.",
|
||||
"prerequisites": [
|
||||
"Reviewed independent vulnerable/fixed/benign push pair required before hook admission."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA-BOOKS",
|
||||
"base": "vampi-regression",
|
||||
"tip": "vampi-regression",
|
||||
"label": "regression",
|
||||
"role": "regression",
|
||||
"capability": "python/flask/access_control",
|
||||
"provenance": "Existing recipe known target, full target population retained; environment vulnerable=1. Snapshot regression, not an introducing push.",
|
||||
"prerequisites": [
|
||||
"Reviewed independent vulnerable/fixed/benign push pair required before hook admission."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "libpng-CVE-2025-66293",
|
||||
"base": "libpng-envelope",
|
||||
"tip": "libpng-envelope",
|
||||
"label": "unsupported",
|
||||
"role": "envelope",
|
||||
"capability": "c/libpng/memory_safety",
|
||||
"provenance": "Arithmetic/bounds is outside the current abstract domain; measures graph/cost envelope only.",
|
||||
"prerequisites": []
|
||||
},
|
||||
{
|
||||
"id": "ghost-reserved-population",
|
||||
"base": "ghost-reserved",
|
||||
"tip": "ghost-reserved",
|
||||
"label": "unreviewed",
|
||||
"role": "reserved_holdout",
|
||||
"capability": "javascript/express/unreviewed",
|
||||
"provenance": "Reserved from production API/module/async shape and public security-fix history before scanner results. Manifest6.64.0-rc.0, Express4.22.2. Not an ESM or Express5 coverage claim.",
|
||||
"prerequisites": [
|
||||
"Review and freeze supported positive/fixed/benign changes from independent history; select family before scanning.",
|
||||
"Freeze core and existing facts after development; preserve reservation and record any contamination."
|
||||
]
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
"version": "pre-push-diagnostic-v1-2026-09-12",
|
||||
"ranking_mode": "evidence",
|
||||
"engine": "joern-4.0.625",
|
||||
"facts": "642d643d5b40a20fe482ace566cc77d132870930ecd2f2e881dcecd7dc9a0641",
|
||||
"queries": "64c1534edb07e770ab8631b48bc156ef54b71ddb8d8ee4357fd5022eb11f0229",
|
||||
"policy": "diagnostic-only-no-admission-v1",
|
||||
"scope": "first-party-only; declared vendor exclusion in graph and targets; exact selected/deferred question IDs required per record",
|
||||
"hardware": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39",
|
||||
"budget_seconds": 30,
|
||||
"core": "7ac2bc93b35ce9d0cdcfc543266025519999ff45339657d4a5d283c67957d682",
|
||||
"targets": {
|
||||
"pmpro-introduce": {
|
||||
"path": "classes/class.memberorder.php",
|
||||
"function": "getMemberOrderByCode",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
936
|
||||
]
|
||||
},
|
||||
"pmpro-repair": {
|
||||
"path": "classes/class.memberorder.php",
|
||||
"function": "getMemberOrderByCode",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
937
|
||||
]
|
||||
},
|
||||
"pmpro-benign": {
|
||||
"path": "classes/class.memberorder.php",
|
||||
"function": "getMemberOrderByCode",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
937
|
||||
]
|
||||
},
|
||||
"nodegoat-introduce": {
|
||||
"path": "app/routes/contributions.js",
|
||||
"function": "handleContributionsUpdate",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
32,
|
||||
33,
|
||||
34
|
||||
]
|
||||
},
|
||||
"nodegoat-repair": {
|
||||
"path": "app/routes/contributions.js",
|
||||
"function": "handleContributionsUpdate",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
32,
|
||||
33,
|
||||
34
|
||||
]
|
||||
},
|
||||
"nodegoat-benign": {
|
||||
"path": "app/routes/contributions.js",
|
||||
"function": "handleContributionsUpdate",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
32,
|
||||
33,
|
||||
34
|
||||
]
|
||||
},
|
||||
"VAmPI-SQLI": {
|
||||
"path": "models/user_model.py",
|
||||
"function": "get_user",
|
||||
"family": "injection",
|
||||
"lines": [
|
||||
73
|
||||
]
|
||||
},
|
||||
"VAmPI-BOLA": {
|
||||
"path": "api_views/users.py",
|
||||
"function": "update_password",
|
||||
"family": "access_control",
|
||||
"lines": [
|
||||
187
|
||||
]
|
||||
},
|
||||
"VAmPI-BOLA-BOOKS": {
|
||||
"path": "api_views/books.py",
|
||||
"function": "get_by_title",
|
||||
"family": "access_control",
|
||||
"lines": [
|
||||
51
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"gates": {
|
||||
"precision": 0.95,
|
||||
"recall": 0.9,
|
||||
"completion": 0.95,
|
||||
"warm_p95_seconds": 30.0,
|
||||
"fixed_false_alerts": 0,
|
||||
"benign_false_alerts": 0
|
||||
},
|
||||
"profile_sha256": "95c26cf474573766fa24c44f4375cfcb506b196412cd421deadff35b4eb9ef3f"
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"profile_sha256": "95c26cf474573766fa24c44f4375cfcb506b196412cd421deadff35b4eb9ef3f",
|
||||
"context": {
|
||||
"ranking_mode": "evidence",
|
||||
"engine": "joern-4.0.625",
|
||||
"facts": "642d643d5b40a20fe482ace566cc77d132870930ecd2f2e881dcecd7dc9a0641",
|
||||
"queries": "64c1534edb07e770ab8631b48bc156ef54b71ddb8d8ee4357fd5022eb11f0229",
|
||||
"policy": "diagnostic-only-no-admission-v1",
|
||||
"scope": "first-party-only; declared vendor exclusion in graph and targets; exact selected/deferred question IDs required per record",
|
||||
"hardware": "Linux-6.8.0-138-generic-x86_64-with-glibc2.39",
|
||||
"budget_seconds": 30,
|
||||
"core": "7ac2bc93b35ce9d0cdcfc543266025519999ff45339657d4a5d283c67957d682"
|
||||
},
|
||||
"records": []
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
[
|
||||
{
|
||||
"schema_version": 2,
|
||||
"measurement": "positional_coverage_not_detection",
|
||||
"ranking_mode": "static",
|
||||
"profile": {
|
||||
"checkout": {
|
||||
"commit": "f16052dce83f05847133ec98f01c5193a41de7d8",
|
||||
"clean": true,
|
||||
"status_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"version": "position-v2",
|
||||
"requested_ranking_mode": "static",
|
||||
"known_targets": [
|
||||
{
|
||||
"id": "VAmPI-SQLI",
|
||||
"family": "injection",
|
||||
"file": "models/user_model.py",
|
||||
"function": "get_user",
|
||||
"line": 73,
|
||||
"in_scope": true
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA",
|
||||
"family": "access_control",
|
||||
"file": "api_views/users.py",
|
||||
"function": "update_password",
|
||||
"line": 187,
|
||||
"in_scope": true
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA-BOOKS",
|
||||
"family": "access_control",
|
||||
"file": "api_views/books.py",
|
||||
"function": "get_by_title",
|
||||
"line": 51,
|
||||
"in_scope": true
|
||||
}
|
||||
],
|
||||
"input_witnesses": [
|
||||
{
|
||||
"path": "api_views/books.py",
|
||||
"bytes_read": 3043,
|
||||
"sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8"
|
||||
},
|
||||
{
|
||||
"path": "api_views/users.py",
|
||||
"bytes_read": 9257,
|
||||
"sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c"
|
||||
},
|
||||
{
|
||||
"path": "models/user_model.py",
|
||||
"bytes_read": 3642,
|
||||
"sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b"
|
||||
}
|
||||
],
|
||||
"implementation": {
|
||||
"benchmarks/ranking/position.py": "c362e58843e669b68bb032574beff2342bd9597e0b451b38c727c1af643450e2",
|
||||
"src/openultrasast/model/regions.py": "0ea1a7b153fad403848aa03f54ae45ea0dcf9c3d8b0f710238362de05139c84b",
|
||||
"src/openultrasast/model/scan.py": "e11423755be4b72cfb6b5fd86b3d2fef68c0cc33aab997210f51430570256e93",
|
||||
"src/openultrasast/model/evidence.py": "cda3619170a77bb8a4b7bb7fecb261e990699fe671714ccc830505a81029788d",
|
||||
"src/openultrasast/model/layout.py": "dc9dc338dff5f1d4389d59d84ba4ddd249949470f9c2303df5ee70ad5d257f2d"
|
||||
},
|
||||
"detection": "not_measured",
|
||||
"sha256": "273dd99adcd94ec93df5e4c30018c145092141ba2c4e28a409d8bb5f8e6939a2"
|
||||
},
|
||||
"detection_recall": null,
|
||||
"supported_population": 3,
|
||||
"unmatched_supported": 1,
|
||||
"input_witnesses": [
|
||||
{
|
||||
"path": "api_views/books.py",
|
||||
"bytes_read": 3043,
|
||||
"sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8"
|
||||
},
|
||||
{
|
||||
"path": "api_views/users.py",
|
||||
"bytes_read": 9257,
|
||||
"sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c"
|
||||
},
|
||||
{
|
||||
"path": "models/user_model.py",
|
||||
"bytes_read": 3642,
|
||||
"sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b"
|
||||
}
|
||||
],
|
||||
"input_status": "readable",
|
||||
"total_seconds": 0.793,
|
||||
"repo": "vampi",
|
||||
"regions": 25,
|
||||
"distinct_ranks": 5,
|
||||
"largest_tier_share": 0.36,
|
||||
"largest_tier_rank": 1.0,
|
||||
"budget_at_recall": null,
|
||||
"budget_at_full_target_position_coverage": null,
|
||||
"known": [
|
||||
{
|
||||
"id": "VAmPI-SQLI",
|
||||
"family": "injection",
|
||||
"site": "models/user_model.py:73:get_user",
|
||||
"position": null,
|
||||
"rank": null,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false,
|
||||
"detection": "not_measured",
|
||||
"position_status": "unresolved_target"
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA",
|
||||
"family": "access_control",
|
||||
"site": "api_views/users.py:187:update_password",
|
||||
"position": 15,
|
||||
"rank": 0.8,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false,
|
||||
"detection": "not_measured",
|
||||
"position_status": "matched"
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA-BOOKS",
|
||||
"family": "access_control",
|
||||
"site": "api_views/books.py:51:get_by_title",
|
||||
"position": 10,
|
||||
"rank": 0.8,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false,
|
||||
"detection": "not_measured",
|
||||
"position_status": "matched"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,20 @@
|
||||
# Historical ranking measurements
|
||||
|
||||
The eight JSON artifacts dated 2026-09-10 and 2026-09-11 are preserved as recorded.
|
||||
They predate the scoring correction in pre-push-safety-net task 1.3 and must not be
|
||||
interpreted as end-to-end detection or pre-push acceptance results.
|
||||
|
||||
An audit on 2026-09-12 read all eight artifacts. Each records VAmPI-SQLI with
|
||||
`position: null`, `reached_transitively: true` and a repository `budget_at_recall`
|
||||
of 16. The old position instrument assigned that transitive label solely because
|
||||
it could not locate a matching region, then omitted the target from the budget
|
||||
calculation. These artifacts contain no queried witness establishing that label.
|
||||
This does not negate separately demonstrated VAmPI detections; it means this
|
||||
particular metric cannot establish them.
|
||||
|
||||
Other recorded positions, scores, and elapsed stage costs remain historical
|
||||
observations under their recorded setup. Rank coverage is not detection recall,
|
||||
and a stage duration is not total hook latency. Use the corrected instrument and
|
||||
an explicit frozen profile for new comparisons. Retain unknown and unanswered
|
||||
cases, and require a queried target witness before counting transitive detection.
|
||||
Do not rewrite these files to make an earlier run appear to use the new semantics.
|
||||
@@ -1,23 +1,19 @@
|
||||
"""Phase 0 of flow-aware-ranking: where does each pinned CVE sit in the order the scan actually examines?
|
||||
"""Positional coverage baseline v2: ranks are not detection recall.
|
||||
|
||||
The ranker's success metric is not "is the CVE in the budget" -- on Paid Memberships Pro it already is, at
|
||||
position 390 of 4,463 -- but **how small the budget can be while every CVE stays in it**. That number is
|
||||
`budget_at_recall`, and until it is written down nothing later can claim to have improved it.
|
||||
|
||||
This is the baseline. It runs no engine and asks no model: it is the region pipeline exactly as the scan
|
||||
uses it, sorted exactly as the scan sorts it, with each known vulnerability's position read off. Offline,
|
||||
against pinned checkouts, so two runs of one repository agree.
|
||||
|
||||
ousast-venv/bin/python benchmarks/ranking/position.py # every pinned recipe with a checkout
|
||||
ousast-venv/bin/python benchmarks/ranking/position.py --json # the same, as an artifact
|
||||
Every declared target remains visible, including unmatched sites. No transitive detection
|
||||
is inferred here: that requires recorded answered queries and target witnesses in the
|
||||
push scorer. Historical v1 artifacts remain unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
@@ -101,10 +97,55 @@ def evidence_ordered_regions(root: Path, regions: list[ScanRegion], language: st
|
||||
|
||||
|
||||
def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidence: bool = False, language: str = "") -> dict[str, object]:
|
||||
regions = ordered_regions(root)
|
||||
started = time.monotonic()
|
||||
inputs = []
|
||||
for path in sorted({str(item["file"]) for item in known}):
|
||||
try:
|
||||
content = (root / path).read_bytes()
|
||||
if not content:
|
||||
raise ValueError("empty source")
|
||||
inputs.append(dict(path=path, bytes_read=len(content), sha256=hashlib.sha256(content).hexdigest()))
|
||||
except (OSError, ValueError) as exc:
|
||||
inputs.append(dict(path=path, bytes_read=0, problem=str(exc)))
|
||||
facts: dict[str, object] = {}
|
||||
# Frozen identity recorded before the evidence pass; this is a rank-only experiment.
|
||||
source_root = Path(__file__).resolve().parents[2]
|
||||
version_paths = [
|
||||
"benchmarks/ranking/position.py",
|
||||
"src/openultrasast/model/regions.py",
|
||||
"src/openultrasast/model/scan.py",
|
||||
"src/openultrasast/model/evidence.py",
|
||||
"src/openultrasast/model/layout.py",
|
||||
]
|
||||
checkout = {}
|
||||
try:
|
||||
commit = (
|
||||
subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, timeout=10).decode().strip()
|
||||
)
|
||||
status = subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], stderr=subprocess.DEVNULL, timeout=10)
|
||||
checkout = dict(commit=commit, clean=not status, status_sha256=hashlib.sha256(status).hexdigest())
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
checkout = dict(problem="checkout commit unavailable")
|
||||
profile = dict(
|
||||
checkout=checkout,
|
||||
version="position-v2",
|
||||
requested_ranking_mode="evidence" if by_evidence else "static",
|
||||
known_targets=known,
|
||||
input_witnesses=inputs,
|
||||
implementation={path: hashlib.sha256((source_root / path).read_bytes()).hexdigest() for path in version_paths},
|
||||
detection="not_measured",
|
||||
)
|
||||
profile["sha256"] = hashlib.sha256(json.dumps(profile, sort_keys=True).encode()).hexdigest()
|
||||
regions = ordered_regions(root) if root.is_dir() else []
|
||||
actual_mode = "static"
|
||||
if by_evidence:
|
||||
regions, facts = evidence_ordered_regions(root, regions, language)
|
||||
try:
|
||||
regions, facts = evidence_ordered_regions(root, regions, language)
|
||||
actual_mode = "evidence"
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError, SystemExit) as exc:
|
||||
regions = []
|
||||
actual_mode = "unavailable"
|
||||
facts = dict(evidence_failure=str(exc))
|
||||
tiers = Counter(r.rank for r in regions)
|
||||
largest = max(tiers.values()) if tiers else 0
|
||||
rows: list[dict[str, object]] = []
|
||||
@@ -112,7 +153,11 @@ def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidenc
|
||||
# A known vulnerability names a file and a function. Match on both, path by suffix because a recipe
|
||||
# writes it relative to the repository and a region may carry it the same way or deeper.
|
||||
hit = next(
|
||||
(i for i, r in enumerate(regions) if r.path.endswith(str(item["file"])) and r.function == item["function"]),
|
||||
(
|
||||
i
|
||||
for i, r in enumerate(regions)
|
||||
if r.path in (str(item["file"]), str(root / str(item["file"]))) and r.function == item["function"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
rows.append(
|
||||
@@ -120,26 +165,39 @@ def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidenc
|
||||
"id": item["id"],
|
||||
"family": item["family"],
|
||||
"site": f"{item['file']}:{item['line']}:{item['function']}",
|
||||
# Zero-based. None means no region is named after this function: the site is a SINK that an
|
||||
# entry-point region elsewhere reaches through `callDepth` -- VAmPI's SQL injection lives in
|
||||
# `models/user_model.py:get_user` and is found from the handler that calls it. Its true
|
||||
# position is that handler's, which only the flow can say, so it is reported and left out of
|
||||
# `budget_at_recall` rather than counted as a miss it is not.
|
||||
# Zero-based position is only a region match, never a completed detection.
|
||||
"position": hit,
|
||||
"rank": regions[hit].rank if hit is not None else None,
|
||||
"in_scope": item["in_scope"],
|
||||
"reached_transitively": hit is None,
|
||||
"reached_transitively": False,
|
||||
"detection": "not_measured",
|
||||
"position_status": "matched" if hit is not None else "unresolved_target",
|
||||
}
|
||||
)
|
||||
positioned = [r["position"] for r in rows if r["position"] is not None and r["in_scope"]]
|
||||
supported = [row for row in rows if row["in_scope"]]
|
||||
all_located = bool(supported) and len(positioned) == len(supported)
|
||||
return {
|
||||
"schema_version": 2,
|
||||
"measurement": "positional_coverage_not_detection",
|
||||
"ranking_mode": actual_mode,
|
||||
"profile": profile,
|
||||
"detection_recall": None,
|
||||
"supported_population": len(supported),
|
||||
"unmatched_supported": len(supported) - len(positioned),
|
||||
"input_witnesses": inputs,
|
||||
"input_status": "readable" if inputs and all(row["bytes_read"] for row in inputs) else "incomplete",
|
||||
"total_seconds": round(time.monotonic() - started, 3),
|
||||
"repo": name,
|
||||
"regions": len(regions),
|
||||
"distinct_ranks": len(tiers),
|
||||
"largest_tier_share": round(largest / len(regions), 3) if regions else 0.0,
|
||||
"largest_tier_rank": max(tiers, key=lambda k: tiers[k]) if tiers else None,
|
||||
# The smallest examined-region budget that still holds every in-scope known vulnerability.
|
||||
"budget_at_recall": (max(positioned) + 1) if positioned else None,
|
||||
"budget_at_recall": None, # Retired misleading name: rank does not measure recall.
|
||||
"budget_at_full_target_position_coverage": (max(positioned) + 1)
|
||||
if all_located and all(row["bytes_read"] for row in inputs)
|
||||
else None,
|
||||
"known": rows,
|
||||
**facts,
|
||||
}
|
||||
@@ -160,8 +218,7 @@ def main() -> int:
|
||||
continue
|
||||
root = checkout_path(recipe)
|
||||
if not root.is_dir():
|
||||
print(f"{recipe.name}: not checked out, skipping", file=sys.stderr)
|
||||
continue
|
||||
print(f"{recipe.name}: not checked out; retaining unresolved targets", file=sys.stderr)
|
||||
known = [
|
||||
{"id": k.id, "family": k.family, "file": k.file, "function": k.function, "line": k.line, "in_scope": k.in_scope}
|
||||
for k in recipe.known
|
||||
@@ -173,20 +230,20 @@ def main() -> int:
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(results, indent=2))
|
||||
return 0
|
||||
return 1 if any(r["unmatched_supported"] or r["input_status"] != "readable" or r.get("evidence_failure") for r in results) else 0
|
||||
for r in results:
|
||||
print(
|
||||
f"{r['repo']}: {r['regions']} regions, {r['distinct_ranks']} distinct ranks, "
|
||||
f"{100 * float(r['largest_tier_share']):.1f}% share rank {r['largest_tier_rank']}"
|
||||
)
|
||||
for k in r["known"]:
|
||||
where = "reached via another region" if k["position"] is None else f"position {k['position']} (rank {k['rank']})"
|
||||
where = "unresolved target position" if k["position"] is None else f"position {k['position']} (rank {k['rank']})"
|
||||
scope = "" if k["in_scope"] else " [out of scope]"
|
||||
print(f" {k['id']:16} {where:28} {k['site']}{scope}")
|
||||
print(f" budget_at_recall = {r['budget_at_recall']}")
|
||||
print(f" budget_at_full_target_position_coverage = {r['budget_at_full_target_position_coverage']}; detection not measured")
|
||||
if "tier_counts" in r:
|
||||
print(f" build {r['build_seconds']}s, evidence {r['evidence_seconds']}s over {r['pairs']} pairs, tiers {r['tier_counts']}")
|
||||
return 0
|
||||
return 1 if any(r["unmatched_supported"] or r["input_status"] != "readable" or r.get("evidence_failure") for r in results) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user