bench(harnessx-removal): deterministic outputs byte-identical after the removal (task 7)

Reran benchmarks/harnessx_removal_equality.py record from a git archive
export of b9afb66 (tasks 1-6 landed; same venv freeze hash as the
baseline, harnessx not importable, no API keys, empty config) and
compared it against 2026-09-30-harnessx-removal-baseline:

  equal=True differences=0

All 13 output files (pairs.json, benchmark, standard scan, improve
--dry-run) are raw-byte identical to the baseline. The suite map
(suite-map.json) accounts for 29 deleted ids, 2 renamed ids and 83
added ids; each deleted or renamed id cites the design's Tests table and
commit cbadc38 or b9afb66. Every surviving test id keeps its outcome
(1740 passed, 38 skipped incl. the README xfail).

Instrument note: a first record failed test_push_hook twice because the
recording shell's PATH lacked .venv/bin (the hook execs `ousast`); the
same two tests fail at b9afb66 without it and pass with it. The
committed record has .venv/bin on PATH.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 12:40:57 +02:00
co-authored by Claude Opus 5.5
parent b9afb66250
commit 800204de87
19 changed files with 21046 additions and 1 deletions
@@ -0,0 +1,630 @@
{
"findings": [
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-flask-debug:app.py:81",
"function_name": null,
"line": 81,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 1.5,
"rationale": "Static pattern python-flask-debug (CWE-489) matched debug on line 81; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"insecure_default",
"syscall_entry"
],
"title": "Flask debug mode enabled needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-os-command:app.py:17",
"function_name": "ping",
"line": 17,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-os-command (CWE-78) matched os.system on line 17; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/ping')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 17,
"function_name": "ping",
"kind": "route",
"line": 15,
"name": "ping",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Python OS command execution needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-path-traversal:app.py:55",
"function_name": "read",
"line": 55,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-path-traversal (CWE-22) matched open on line 55; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/read')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 55,
"function_name": "read",
"kind": "route",
"line": 54,
"name": "read",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Python file open with external input needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-shell-true:app.py:23",
"function_name": "run",
"line": 23,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-shell-true (CWE-78) matched subprocess.check_output on line 23; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/run')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 23,
"function_name": "run",
"kind": "route",
"line": 21,
"name": "run",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Subprocess shell execution needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:44",
"function_name": "user",
"line": 44,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-sql-injection (CWE-89) matched .execute on line 44; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/user')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 44,
"function_name": "user",
"kind": "route",
"line": 42,
"name": "user",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"injection",
"syscall_entry"
],
"title": "Python SQL string composition needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:50",
"function_name": "report",
"line": 50,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-sql-injection (CWE-89) matched .execute on line 50; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/report')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 50,
"function_name": "report",
"kind": "route",
"line": 48,
"name": "report",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"injection",
"syscall_entry"
],
"title": "Python SQL string composition needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-ssrf:app.py:60",
"function_name": "fetch",
"line": 60,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-ssrf (CWE-918) matched requests.get on line 60; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/fetch')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 60,
"function_name": "fetch",
"kind": "route",
"line": 59,
"name": "fetch",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"network_entry",
"syscall_entry"
],
"title": "Python outbound request with external input needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-ssti:app.py:70",
"function_name": "render",
"line": 70,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-ssti (CWE-94) matched render_template_string on line 70; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/render')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 70,
"function_name": "render",
"kind": "route",
"line": 69,
"name": "render",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"injection",
"syscall_entry"
],
"title": "Python server-side template injection needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:33",
"function_name": "load",
"line": 33,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-unsafe-deserialization (CWE-502) matched pickle.loads on line 33; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/load')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 33,
"function_name": "load",
"kind": "route",
"line": 32,
"name": "load",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Unsafe Python deserialization needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:38",
"function_name": "config",
"line": 38,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-unsafe-deserialization (CWE-502) matched yaml.load on line 38; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/config')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 38,
"function_name": "config",
"kind": "route",
"line": 37,
"name": "config",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Unsafe Python deserialization needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-eval:app.py:28",
"function_name": "calc",
"line": 28,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-unsafe-eval (CWE-95) matched eval on line 28; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/calc')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 28,
"function_name": "calc",
"kind": "route",
"line": 27,
"name": "calc",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Dynamic Python execution needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-weak-hash:app.py:65",
"function_name": "token",
"line": 65,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-weak-hash (CWE-327) matched md5 on line 65; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/token')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 65,
"function_name": "token",
"kind": "route",
"line": 64,
"name": "token",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Python weak hash needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:app.py:77:execute",
"function_name": null,
"line": 77,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.0,
"rationale": "uninventoried sink execute reached from request",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"overlay-originated"
],
"title": "Overlay coverage: execute"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:safe.py:11:os.system",
"function_name": null,
"line": 11,
"path": "safe.py",
"proposed_snippet": null,
"ranking_priority": 0.0,
"rationale": "uninventoried sink os.system reached from parameter",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"overlay-originated"
],
"title": "Overlay coverage: os.system"
},
{
"confidence": "low",
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:44:identity_constraint",
"function_name": "user",
"line": 44,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.3,
"rationale": "protected_read on users in user is reached without a dominating identity_constraint. Why the obligation exists: the operation fact alone (function-local mode). Label: function_local.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"obligation:protected_read",
"discharger:identity_constraint",
"obligation_evidence:function_local",
"resource:users"
],
"title": "Obligation not discharged: protected_read on users without identity_constraint"
},
{
"confidence": "low",
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:50:identity_constraint",
"function_name": "report",
"line": 50,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.3,
"rationale": "protected_read on reports in report is reached without a dominating identity_constraint. Why the obligation exists: the operation fact alone (function-local mode). Label: function_local.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"obligation:protected_read",
"discharger:identity_constraint",
"obligation_evidence:function_local",
"resource:reports"
],
"title": "Obligation not discharged: protected_read on reports without identity_constraint"
},
{
"confidence": "low",
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:77:identity_constraint",
"function_name": "search",
"line": 77,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.3,
"rationale": "protected_read on a resource in search is reached without a dominating identity_constraint. Why the obligation exists: the operation fact alone (function-local mode). Label: function_local.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"obligation:protected_read",
"discharger:identity_constraint",
"obligation_evidence:function_local"
],
"title": "Obligation not discharged: protected_read on a resource without identity_constraint"
}
]
}
@@ -0,0 +1,192 @@
[
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-os-command:app.py:17",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=critical, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:critical"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-shell-true:app.py:23",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=critical, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:critical"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:44",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=high, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:high"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:50",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=high, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:high"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-unsafe-eval:app.py:28",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=critical, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:critical"
],
"votes": {
"accepted": 2
},
"warnings": []
}
]
@@ -0,0 +1,11 @@
{
"contract_mode": "strict",
"model_roles": {
"hunter": null,
"patcher": null,
"ranker": null,
"verifier": null
},
"processor_versions": {},
"prompt_hashes": {}
}
@@ -0,0 +1,461 @@
{
"artifacts": {
"complexity_map": "complexity_map.json",
"findings": "findings.json",
"fusion": "fusion.json",
"markdown": "report.md",
"overlay": "overlay.json",
"sarif": "report.sarif",
"score": "score.json",
"trajectories": "traces/hunter_trajectories.jsonl",
"verification": "verification.json"
},
"complexity": {
"heuristic_only": true,
"hotspot_count": 14
},
"degradations": [
{
"detail": "no path records; obligations evaluated within entry-point handlers only",
"files": [
"app.py",
"safe.py"
],
"language": "python",
"reason": "obligations_function_local",
"stage": "obligations"
},
{
"detail": "no hunter client or model",
"reason": "intent_adjudication_unavailable",
"stage": "obligations"
},
{
"reason": "cpg_unavailable",
"stage": "model"
},
{
"reason": "hunter_model_unavailable",
"stage": "map"
}
],
"findings": [
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-flask-debug:app.py:81",
"line": 81,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "rejected"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-os-command:app.py:17",
"line": 17,
"path": "app.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-path-traversal:app.py:55",
"line": 55,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-shell-true:app.py:23",
"line": 23,
"path": "app.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:44",
"line": 44,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:50",
"line": 50,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-ssrf:app.py:60",
"line": 60,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-ssti:app.py:70",
"line": 70,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:33",
"line": 33,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:38",
"line": 38,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-eval:app.py:28",
"line": 28,
"path": "app.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-weak-hash:app.py:65",
"line": 65,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:app.py:77:execute",
"line": 77,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:safe.py:11:os.system",
"line": 11,
"path": "safe.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:44:identity_constraint",
"line": 44,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:50:identity_constraint",
"line": 50,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:77:identity_constraint",
"line": 77,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
}
],
"fusion": [
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-os-command:app.py:17",
"triggers": [
"gates_risky_fix",
"severity:critical"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-shell-true:app.py:23",
"triggers": [
"gates_risky_fix",
"severity:critical"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:44",
"triggers": [
"gates_risky_fix",
"severity:high"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:50",
"triggers": [
"gates_risky_fix",
"severity:high"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-unsafe-eval:app.py:28",
"triggers": [
"gates_risky_fix",
"severity:critical"
]
}
],
"model": {
"by_rung": {},
"cost_usd": 0.0,
"coverage": [],
"family_coverage": [],
"model_calls": 0,
"partitions": [],
"query_seconds_by_kind": {},
"question_outcomes": [],
"regions_scanned": 0,
"regions_unasked": 0,
"regions_unjudged": 0,
"scope": null,
"seconds": 0.0,
"skipped": "cpg_unavailable",
"unjudged_sample": []
},
"obligations": {
"degradations": [
{
"detail": "no path records; obligations evaluated within entry-point handlers only",
"files": [
"app.py",
"safe.py"
],
"language": "python",
"reason": "obligations_function_local",
"stage": "obligations"
},
{
"detail": "no hunter client or model",
"reason": "intent_adjudication_unavailable",
"stage": "obligations"
}
],
"findings_by_label": {
"function_local": 3
},
"operations": 4,
"policy_version": null,
"sets": [
{
"handlers": 1,
"module": "app",
"resource": "reports"
},
{
"handlers": 1,
"module": "app",
"resource": "users"
}
],
"sibling_sets": 2,
"under_populated": 2
},
"provenance": {
"agent_commits": 0,
"sampled_commits": 0,
"signals": [
"git:unavailable"
],
"value": "human"
},
"score": {
"by_category": {
"Code Injection": 2,
"Code Quality": 1,
"Command or Argument Injection": 2,
"Cryptographic Issues": 1,
"Directory Traversal": 1,
"Encapsulation": 2,
"Information Leakage": 1,
"SQL Injection": 2
},
"gate": {
"block_severity_reachable": 5,
"blocking": false,
"min_score": 80,
"passed": false
},
"max_severity": 5,
"out_of_scope_dynamic_only": 0,
"penalty_total": 105.6,
"project_score": 17,
"unmapped_cwe": []
},
"stages": {
"completed": [
"static",
"map"
],
"requested": [
"static",
"map"
],
"skipped": []
},
"target": "<ROOT>/benchmarks/fixtures/python-vulnerable"
}
@@ -0,0 +1,6 @@
scan_id=<RUN_ID>
run_dir=<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>
file_targets=2
ranked_targets=2
findings=17
calibrations_applied=1
@@ -0,0 +1,280 @@
{
"verifications": [
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "No function-level attacker-reachable entry point currently reaches this line.",
"evidence_level": "static_corroboration",
"finding_id": "python-flask-debug:app.py:81",
"pro_case": "A static pattern matched the source line.",
"required_next_step": "Add reachability evidence before prioritizing a fix.",
"status": "rejected",
"tie_breaker": "Provide call-graph, route, CLI, parser, or dynamic evidence linking attacker input to this function.",
"verified": false
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-os-command:app.py:17",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-path-traversal:app.py:55",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-shell-true:app.py:23",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:44",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:50",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-ssrf:app.py:60",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-ssti:app.py:70",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:33",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:38",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-eval:app.py:28",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-weak-hash:app.py:65",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
}
]
}