mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-22 22:49:29 +00:00
bench(harnessx-removal): deterministic outputs byte-identical after the removal (task 7)
Reran benchmarks/harnessx_removal_equality.py record from a git archive export ofb9afb66(tasks 1-6 landed; same venv freeze hash as the baseline, harnessx not importable, no API keys, empty config) and compared it against 2026-09-30-harnessx-removal-baseline: equal=True differences=0 All 13 output files (pairs.json, benchmark, standard scan, improve --dry-run) are raw-byte identical to the baseline. The suite map (suite-map.json) accounts for 29 deleted ids, 2 renamed ids and 83 added ids; each deleted or renamed id cites the design's Tests table and commitcbadc38orb9afb66. Every surviving test id keeps its outcome (1740 passed, 38 skipped incl. the README xfail). Instrument note: a first record failed test_push_hook twice because the recording shell's PATH lacked .venv/bin (the hook execs `ousast`); the same two tests fail atb9afb66without it and pass with it. The committed record has .venv/bin on PATH. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b9afb66250
commit
800204de87
+192
@@ -0,0 +1,192 @@
|
||||
[
|
||||
{
|
||||
"decision_source": "deterministic-reconciler",
|
||||
"degradations": [],
|
||||
"disposition": "accepted",
|
||||
"finding_id": "python-os-command:app.py:17",
|
||||
"model_ids": [],
|
||||
"panels": [
|
||||
{
|
||||
"confidence": 0.75,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
|
||||
"leaning": "vulnerability",
|
||||
"model_id": null,
|
||||
"rationale": "reachability=reachable, severity=critical, verified=True",
|
||||
"role": "panel_a",
|
||||
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
|
||||
},
|
||||
{
|
||||
"confidence": 0.6,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
|
||||
"leaning": "false_positive",
|
||||
"model_id": null,
|
||||
"rationale": "reachable=True, rejected=False, verified=True",
|
||||
"role": "panel_b",
|
||||
"vulnerability_case": "A tainted sink shape exists in the source."
|
||||
}
|
||||
],
|
||||
"triggered": true,
|
||||
"triggers": [
|
||||
"gates_risky_fix",
|
||||
"severity:critical"
|
||||
],
|
||||
"votes": {
|
||||
"accepted": 2
|
||||
},
|
||||
"warnings": []
|
||||
},
|
||||
{
|
||||
"decision_source": "deterministic-reconciler",
|
||||
"degradations": [],
|
||||
"disposition": "accepted",
|
||||
"finding_id": "python-shell-true:app.py:23",
|
||||
"model_ids": [],
|
||||
"panels": [
|
||||
{
|
||||
"confidence": 0.75,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
|
||||
"leaning": "vulnerability",
|
||||
"model_id": null,
|
||||
"rationale": "reachability=reachable, severity=critical, verified=True",
|
||||
"role": "panel_a",
|
||||
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
|
||||
},
|
||||
{
|
||||
"confidence": 0.6,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
|
||||
"leaning": "false_positive",
|
||||
"model_id": null,
|
||||
"rationale": "reachable=True, rejected=False, verified=True",
|
||||
"role": "panel_b",
|
||||
"vulnerability_case": "A tainted sink shape exists in the source."
|
||||
}
|
||||
],
|
||||
"triggered": true,
|
||||
"triggers": [
|
||||
"gates_risky_fix",
|
||||
"severity:critical"
|
||||
],
|
||||
"votes": {
|
||||
"accepted": 2
|
||||
},
|
||||
"warnings": []
|
||||
},
|
||||
{
|
||||
"decision_source": "deterministic-reconciler",
|
||||
"degradations": [],
|
||||
"disposition": "accepted",
|
||||
"finding_id": "python-sql-injection:app.py:44",
|
||||
"model_ids": [],
|
||||
"panels": [
|
||||
{
|
||||
"confidence": 0.75,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
|
||||
"leaning": "vulnerability",
|
||||
"model_id": null,
|
||||
"rationale": "reachability=reachable, severity=high, verified=True",
|
||||
"role": "panel_a",
|
||||
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
|
||||
},
|
||||
{
|
||||
"confidence": 0.6,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
|
||||
"leaning": "false_positive",
|
||||
"model_id": null,
|
||||
"rationale": "reachable=True, rejected=False, verified=True",
|
||||
"role": "panel_b",
|
||||
"vulnerability_case": "A tainted sink shape exists in the source."
|
||||
}
|
||||
],
|
||||
"triggered": true,
|
||||
"triggers": [
|
||||
"gates_risky_fix",
|
||||
"severity:high"
|
||||
],
|
||||
"votes": {
|
||||
"accepted": 2
|
||||
},
|
||||
"warnings": []
|
||||
},
|
||||
{
|
||||
"decision_source": "deterministic-reconciler",
|
||||
"degradations": [],
|
||||
"disposition": "accepted",
|
||||
"finding_id": "python-sql-injection:app.py:50",
|
||||
"model_ids": [],
|
||||
"panels": [
|
||||
{
|
||||
"confidence": 0.75,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
|
||||
"leaning": "vulnerability",
|
||||
"model_id": null,
|
||||
"rationale": "reachability=reachable, severity=high, verified=True",
|
||||
"role": "panel_a",
|
||||
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
|
||||
},
|
||||
{
|
||||
"confidence": 0.6,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
|
||||
"leaning": "false_positive",
|
||||
"model_id": null,
|
||||
"rationale": "reachable=True, rejected=False, verified=True",
|
||||
"role": "panel_b",
|
||||
"vulnerability_case": "A tainted sink shape exists in the source."
|
||||
}
|
||||
],
|
||||
"triggered": true,
|
||||
"triggers": [
|
||||
"gates_risky_fix",
|
||||
"severity:high"
|
||||
],
|
||||
"votes": {
|
||||
"accepted": 2
|
||||
},
|
||||
"warnings": []
|
||||
},
|
||||
{
|
||||
"decision_source": "deterministic-reconciler",
|
||||
"degradations": [],
|
||||
"disposition": "accepted",
|
||||
"finding_id": "python-unsafe-eval:app.py:28",
|
||||
"model_ids": [],
|
||||
"panels": [
|
||||
{
|
||||
"confidence": 0.75,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
|
||||
"leaning": "vulnerability",
|
||||
"model_id": null,
|
||||
"rationale": "reachability=reachable, severity=critical, verified=True",
|
||||
"role": "panel_a",
|
||||
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
|
||||
},
|
||||
{
|
||||
"confidence": 0.6,
|
||||
"disposition": "accepted",
|
||||
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
|
||||
"leaning": "false_positive",
|
||||
"model_id": null,
|
||||
"rationale": "reachable=True, rejected=False, verified=True",
|
||||
"role": "panel_b",
|
||||
"vulnerability_case": "A tainted sink shape exists in the source."
|
||||
}
|
||||
],
|
||||
"triggered": true,
|
||||
"triggers": [
|
||||
"gates_risky_fix",
|
||||
"severity:critical"
|
||||
],
|
||||
"votes": {
|
||||
"accepted": 2
|
||||
},
|
||||
"warnings": []
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user