mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(ci): run ruff format, and record run 5 at 90% pair-correct
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.
Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.
Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 90.0%
leak 12.3% 6.7%
detection -- 96.7%
cost ~$0.69 $0.0068
27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).
One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.
The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
14ca0a96ce
commit
fe45081b43
@@ -24,8 +24,7 @@ from pathlib import Path
|
||||
def _spec(): # type: ignore[no-untyped-def]
|
||||
from openultrasast.model.specs import TaintSpec
|
||||
|
||||
return TaintSpec(family="injection", language="python", sources=("request.args",),
|
||||
sinks=("os.system",), sanitizers=("escape",))
|
||||
return TaintSpec(family="injection", language="python", sources=("request.args",), sinks=("os.system",), sanitizers=("escape",))
|
||||
|
||||
|
||||
def _cpg(rows): # type: ignore[no-untyped-def]
|
||||
@@ -46,10 +45,28 @@ class _Client:
|
||||
return ChatResponse(content=self._answer)
|
||||
|
||||
|
||||
ENTAILING = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['c']",
|
||||
"sanitized": False, "length": 3, "inLabeledScope": True}]
|
||||
SANITIZED = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['c']",
|
||||
"sanitized": True, "length": 3, "inLabeledScope": True}]
|
||||
ENTAILING = [
|
||||
{
|
||||
"sink": "os.system(cmd)",
|
||||
"sinkLine": "4",
|
||||
"sinkMethod": "run",
|
||||
"source": "request.args['c']",
|
||||
"sanitized": False,
|
||||
"length": 3,
|
||||
"inLabeledScope": True,
|
||||
}
|
||||
]
|
||||
SANITIZED = [
|
||||
{
|
||||
"sink": "os.system(cmd)",
|
||||
"sinkLine": "4",
|
||||
"sinkMethod": "run",
|
||||
"source": "request.args['c']",
|
||||
"sanitized": True,
|
||||
"length": 3,
|
||||
"inLabeledScope": True,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_an_entailed_site_becomes_a_finding_with_no_model_call() -> None:
|
||||
@@ -85,8 +102,14 @@ def test_a_candidate_the_model_cannot_see_is_reported_at_suspicion() -> None:
|
||||
from openultrasast.model.pipeline import scan_region
|
||||
|
||||
client = _Client()
|
||||
findings = scan_region(_cpg([]), _spec(), function="run", client=client, model="m",
|
||||
candidates=({"id": "app.py:9:raw_query", "text": "db.raw(q)", "line": 9},))
|
||||
findings = scan_region(
|
||||
_cpg([]),
|
||||
_spec(),
|
||||
function="run",
|
||||
client=client,
|
||||
model="m",
|
||||
candidates=({"id": "app.py:9:raw_query", "text": "db.raw(q)", "line": 9},),
|
||||
)
|
||||
assert client.calls == 1, "one bounded question per candidate"
|
||||
assert len(findings) == 1 and findings[0].rung is Rung.SUSPICION
|
||||
assert findings[0].site == "app.py:9:raw_query"
|
||||
@@ -96,8 +119,14 @@ def test_a_candidate_the_judge_declines_is_not_reported() -> None:
|
||||
from openultrasast.model.pipeline import scan_region
|
||||
|
||||
client = _Client('{"vulnerable": false, "why": "constant"}')
|
||||
findings = scan_region(_cpg([]), _spec(), function="run", client=client, model="m",
|
||||
candidates=({"id": "app.py:9:x", "text": "db.raw('literal')", "line": 9},))
|
||||
findings = scan_region(
|
||||
_cpg([]),
|
||||
_spec(),
|
||||
function="run",
|
||||
client=client,
|
||||
model="m",
|
||||
candidates=({"id": "app.py:9:x", "text": "db.raw('literal')", "line": 9},),
|
||||
)
|
||||
assert findings == []
|
||||
|
||||
|
||||
@@ -107,8 +136,9 @@ def test_without_a_client_only_what_the_model_establishes_is_reported() -> None:
|
||||
from openultrasast.model.pipeline import scan_region
|
||||
|
||||
assert scan_region(_cpg(ENTAILING), _spec(), function="run", client=None, model="", candidates=())[0].rung is Rung.ENTAILED
|
||||
assert scan_region(_cpg([]), _spec(), function="run", client=None, model="",
|
||||
candidates=({"id": "a:1:x", "text": "t", "line": 1},)) == []
|
||||
assert (
|
||||
scan_region(_cpg([]), _spec(), function="run", client=None, model="", candidates=({"id": "a:1:x", "text": "t", "line": 1},)) == []
|
||||
)
|
||||
|
||||
|
||||
def test_the_candidate_budget_is_bounded() -> None:
|
||||
@@ -132,8 +162,7 @@ def test_the_candidate_budget_is_bounded() -> None:
|
||||
def _dominance_spec(): # type: ignore[no-untyped-def]
|
||||
from openultrasast.model.specs import DominanceSpec
|
||||
|
||||
return DominanceSpec(family="access_control", language="python",
|
||||
operations=("filter_by",), dischargers=("current_user",))
|
||||
return DominanceSpec(family="access_control", language="python", operations=("filter_by",), dischargers=("current_user",))
|
||||
|
||||
|
||||
def _config_spec(): # type: ignore[no-untyped-def]
|
||||
@@ -146,9 +175,10 @@ def test_an_absence_family_is_arbitrated_by_dominance_not_taint() -> None:
|
||||
from openultrasast.model.ladder import Rung
|
||||
from openultrasast.model.pipeline import scan_region
|
||||
|
||||
rows = [{"operation": "Note.query.filter_by(id=n)", "opLine": "9", "opMethod": "leaky", "dominatingGuards": []},
|
||||
{"operation": "Note.query.filter_by(id=n)", "opLine": "3", "opMethod": "safe",
|
||||
"dominatingGuards": ["current_user.id"]}]
|
||||
rows = [
|
||||
{"operation": "Note.query.filter_by(id=n)", "opLine": "9", "opMethod": "leaky", "dominatingGuards": []},
|
||||
{"operation": "Note.query.filter_by(id=n)", "opLine": "3", "opMethod": "safe", "dominatingGuards": ["current_user.id"]},
|
||||
]
|
||||
findings = scan_region(_cpg(rows), _dominance_spec(), function="leaky", client=None, model="", candidates=())
|
||||
assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED
|
||||
assert findings[0].family == "access_control"
|
||||
@@ -158,8 +188,7 @@ def test_a_configuration_family_is_arbitrated_by_constant_abstraction() -> None:
|
||||
from openultrasast.model.ladder import Rung
|
||||
from openultrasast.model.pipeline import scan_region
|
||||
|
||||
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app",
|
||||
"literalArgs": ['"*"'], "args": ["app"]}]
|
||||
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app"]}]
|
||||
findings = scan_region(_cpg(rows), _config_spec(), function="create_app", client=None, model="", candidates=())
|
||||
assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED
|
||||
assert findings[0].family == "config_secrets"
|
||||
@@ -191,8 +220,10 @@ def test_a_corroborated_absence_finding_survives_a_judge_that_agrees() -> None:
|
||||
from openultrasast.model.ladder import Rung
|
||||
from openultrasast.model.pipeline import scan_region
|
||||
|
||||
rows = [{"operation": "q.filter_by(id=n)", "opLine": "9", "opMethod": "leaky", "dominatingGuards": []},
|
||||
{"operation": "q.filter_by(id=n)", "opLine": "3", "opMethod": "safe", "dominatingGuards": ["current_user"]}]
|
||||
rows = [
|
||||
{"operation": "q.filter_by(id=n)", "opLine": "9", "opMethod": "leaky", "dominatingGuards": []},
|
||||
{"operation": "q.filter_by(id=n)", "opLine": "3", "opMethod": "safe", "dominatingGuards": ["current_user"]},
|
||||
]
|
||||
findings = scan_region(_cpg(rows), _dominance_spec(), function="leaky", client=_Client(), model="m", candidates=())
|
||||
assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED
|
||||
|
||||
|
||||
Reference in New Issue
Block a user