marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 04:29:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): run ruff format, and record run 5 at 90% pair-correct

Browse Source 
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.

Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.

Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:

                       hunter (K=5)   pipeline (K=1)
  pair-correct              76.7%          90.0%
  leak                      12.3%           6.7%
  detection                    --          96.7%
  cost                     ~$0.69        $0.0068

27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).

One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.

The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-08 19:46:58 +02:00
co-authored by Claude Opus 5
parent 14ca0a96ce
commit fe45081b43
24 changed files with 397 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_model_config_value.py
+31 -17
View File
@@ -14,9 +14,9 @@ from pathlib import Path
def _spec(): # type: ignore[no-untyped-def]
from openultrasast.model.specs import ConfigSpec
return ConfigSpec(family="config_secrets", language="python",
settings=("CORS", "set_cookie", "app.run"),
permissive=("'*'", '"*"', "True", "0.0.0.0"))
return ConfigSpec(
family="config_secrets", language="python", settings=("CORS", "set_cookie", "app.run"), permissive=("'*'", '"*"', "True", "0.0.0.0")
)
def _cpg(rows): # type: ignore[no-untyped-def]
@@ -29,8 +29,7 @@ def test_a_permissive_literal_is_entailed() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.ladder import Rung
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app",
"literalArgs": ['"*"'], "args": ["app", '"*"']}]
rows = [{"setting": 'CORS(app, origins="*")', "line": "3", "method": "create_app", "literalArgs": ['"*"'], "args": ["app", '"*"']}]
answer = verdict(_cpg(rows), _spec(), function="create_app")
assert answer is not None and answer.rung is Rung.ENTAILED
assert "*" in answer.witness and "CORS" in answer.witness
@@ -39,8 +38,15 @@ def test_a_permissive_literal_is_entailed() -> None:
def test_a_restrictive_literal_yields_no_finding() -> None:
from openultrasast.model.config_value import verdict
rows = [{"setting": 'CORS(app, origins="https://app.example")', "line": "3", "method": "create_app",
"literalArgs": ['"https://app.example"'], "args": ["app", '"https://app.example"']}]
rows = [
{
"setting": 'CORS(app, origins="https://app.example")',
"line": "3",
"method": "create_app",
"literalArgs": ['"https://app.example"'],
"args": ["app", '"https://app.example"'],
}
]
assert verdict(_cpg(rows), _spec(), function="create_app") is None
@@ -49,8 +55,15 @@ def test_a_non_literal_argument_is_corroborated_not_entailed() -> None:
from openultrasast.model.config_value import verdict
from openultrasast.model.ladder import Rung
rows = [{"setting": "CORS(app, origins=configured_origins)", "line": "3", "method": "create_app",
"literalArgs": [], "args": ["app", "configured_origins"]}]
rows = [
{
"setting": "CORS(app, origins=configured_origins)",
"line": "3",
"method": "create_app",
"literalArgs": [],
"args": ["app", "configured_origins"],
}
]
answer = verdict(_cpg(rows), _spec(), function="create_app")
assert answer is not None and answer.rung is Rung.CORROBORATED
@@ -72,8 +85,7 @@ def test_a_failed_query_is_not_read_as_a_safe_configuration() -> None:
def test_the_verdict_is_deterministic() -> None:
from openultrasast.model.config_value import verdict
rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login",
"literalArgs": ["True", "0.0.0.0"], "args": ["x"]}]
rows = [{"setting": 'set_cookie("s", secure=False)', "line": "9", "method": "login", "literalArgs": ["True", "0.0.0.0"], "args": ["x"]}]
cpg = _cpg(rows)
assert verdict(cpg, _spec(), function="login") == verdict(cpg, _spec(), function="login")
@@ -90,8 +102,12 @@ def test_the_settings_table_covers_the_frameworks_in_the_corpus() -> None:
from openultrasast.model.specs import config_specs
settings = set(config_specs(language="python")["config_secrets"].settings)
for framework, call in (("aiohttp", "session_setup"), ("aiohttp", "EncryptedCookieStorage"),
("django", "SECURE_SSL_REDIRECT"), ("flask", "set_cookie")):
for framework, call in (
("aiohttp", "session_setup"),
("aiohttp", "EncryptedCookieStorage"),
("django", "SECURE_SSL_REDIRECT"),
("flask", "set_cookie"),
):
assert call in settings, f"{framework}'s {call} is not a modelled security setting"
@@ -102,8 +118,7 @@ def test_a_weak_algorithm_is_entailed_by_its_literal() -> None:
from openultrasast.model.specs import config_specs
spec = config_specs(language="python")["config_secrets"]
rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest",
"literalArgs": ['"md5"'], "args": ['"md5"']}]
rows = [{"setting": 'hashlib.new("md5")', "line": "4", "method": "digest", "literalArgs": ['"md5"'], "args": ['"md5"']}]
answer = verdict(_cpg(rows), spec, function="digest")
assert answer is not None and answer.rung is Rung.ENTAILED
assert "md5" in answer.witness.lower()
@@ -114,6 +129,5 @@ def test_a_strong_algorithm_yields_no_finding() -> None:
from openultrasast.model.specs import config_specs
spec = config_specs(language="python")["config_secrets"]
rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest",
"literalArgs": ['"sha256"'], "args": ['"sha256"']}]
rows = [{"setting": 'hashlib.new("sha256")', "line": "4", "method": "digest", "literalArgs": ['"sha256"'], "args": ['"sha256"']}]
assert verdict(_cpg(rows), spec, function="digest") is None