mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(constrained-detector): candidates from the semantic IR, shaped by family
Task 1.1. The detector stops searching: something deterministic decides what
it looks at, ordered by (path, line, name) and bounded by a constant, so the
batches a later task forms are determined by the input alone.
A call candidate carries what the model used to grep for -- the arguments,
and the binding chain the IR resolved for them, nearest first. An access
control candidate is the operation with the guards in scope, because an
absence bug has no sink.
Two things the fixture taught while writing it. The IR emits a chained call
twice at one line, once bare and once carrying the keyword arguments, so
deduplication keeps the richest: a bare `filter_by` and
`filter_by(user_id=owner)` are the same site and only one of them says what
the constraint was. And `leaky` and `constrained` in the shared obligations
fixture both discharge through `filter_by` -- what separates them is that one
chain ends at `request.args.get('owner')` and the other at `request.user.id`.
That contrast is now the test, and it is exactly the judgement the model is
being narrowed down to make.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
d9d6dfdbcf
commit
8d771901bf
@@ -0,0 +1,156 @@
|
||||
"""Candidates come from the semantic IR, shaped by family (constrained-detector 1.1, Req 1.1, 1.2, 1.4-1.6).
|
||||
|
||||
The detector stops searching. Something deterministic decides what it looks at, which makes that enumerator a hard
|
||||
ceiling on recall — measured before this was built at 12.1% for the pattern ruleset against 96.6% for the IR. The
|
||||
ordering is part of the contract: the batches a later task forms, and therefore the sequence of model calls, must
|
||||
be determined by the input alone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_obligation_operations import APP # noqa: E402
|
||||
|
||||
from openultrasast.learning.families import load_families # noqa: E402
|
||||
|
||||
INJECTION = (
|
||||
"from flask import request\n"
|
||||
"import os\n"
|
||||
"import sqlite3\n\n\n"
|
||||
"@app.route('/run')\n"
|
||||
"def run():\n"
|
||||
" raw = request.args.get('cmd')\n"
|
||||
" command = 'echo ' + raw\n"
|
||||
" os.system(command)\n"
|
||||
" return 'ok'\n"
|
||||
)
|
||||
|
||||
|
||||
def _region(path: str = "app.py", function: str | None = "run"): # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.detectors import Region
|
||||
|
||||
return Region(path=path, function=function)
|
||||
|
||||
|
||||
def _write(tmp_path: Path, name: str, text: str) -> Path:
|
||||
(tmp_path / name).write_text(text)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", INJECTION)
|
||||
result = enumerate_candidates(root, _region(), "injection", taxonomy=load_families())
|
||||
assert result.reason == ""
|
||||
sites = {item.name: item for item in result.candidates}
|
||||
assert "os.system" in sites or "system" in sites, sorted(sites)
|
||||
sink = sites.get("os.system") or sites["system"]
|
||||
assert sink.kind == "call" and sink.function == "run" and sink.path == "app.py"
|
||||
assert sink.line == 10 # `from flask` 1, imports 2-3, blanks, decorator 6, def 7, raw 8, command 9
|
||||
assert sink.arg_texts == ("command",)
|
||||
# the chain the IR resolved, nearest first: command <- 'echo ' + raw <- request.args.get('cmd')
|
||||
assert any("raw" in step for step in sink.binding_chain), sink.binding_chain
|
||||
assert any("request" in step for step in sink.binding_chain), sink.binding_chain
|
||||
assert sink.id in {"app.py:10:os.system", "app.py:10:system"}
|
||||
|
||||
|
||||
def test_the_family_decides_which_kinds_are_candidates() -> None:
|
||||
from openultrasast.learning.candidates import FAMILY_SHAPE
|
||||
|
||||
taxonomy = load_families()
|
||||
assert set(FAMILY_SHAPE) == {family.id for family in taxonomy.families}, "every family says what carries its bug"
|
||||
assert FAMILY_SHAPE["injection"] == ("call",)
|
||||
assert FAMILY_SHAPE["access_control"] == ("operation",)
|
||||
assert "bind" in FAMILY_SHAPE["config_secrets"]
|
||||
assert "call" in FAMILY_SHAPE["prototype"]
|
||||
|
||||
|
||||
def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_path: Path) -> None:
|
||||
"""An absence bug has no sink. Its candidate is the operation, and what the model needs is what guards it."""
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", APP)
|
||||
guarded = enumerate_candidates(root, _region(function="guarded"), "access_control", taxonomy=load_families())
|
||||
assert guarded.reason == "" and guarded.candidates
|
||||
assert all(item.kind == "operation" for item in guarded.candidates)
|
||||
assert any("login_required" in guard for item in guarded.candidates for guard in item.guards)
|
||||
# `leaky` and `constrained` both constrain by `filter_by`; what separates them is where the constraining value
|
||||
# came from, and that is the binding chain the IR resolved — the work the model used to do by grepping.
|
||||
leaky = enumerate_candidates(root, _region(function="leaky"), "access_control", taxonomy=load_families())
|
||||
constrained = enumerate_candidates(root, _region(function="constrained"), "access_control", taxonomy=load_families())
|
||||
leaky_site = next(item for item in leaky.candidates if item.name.endswith("filter_by"))
|
||||
good_site = next(item for item in constrained.candidates if item.name.endswith("filter_by"))
|
||||
assert "user_id=owner" in leaky_site.arg_texts and "user_id=user_id" in good_site.arg_texts
|
||||
assert any("request.args.get" in step for step in leaky_site.binding_chain), leaky_site.binding_chain
|
||||
assert any("request.user.id" in step for step in good_site.binding_chain), good_site.binding_chain
|
||||
|
||||
|
||||
def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", "def broken(:\n pass\n")
|
||||
result = enumerate_candidates(root, _region(function="broken"), "injection", taxonomy=load_families())
|
||||
assert result.candidates == () and result.reason == "unsupported_language"
|
||||
|
||||
|
||||
def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", "def empty():\n return 1\n")
|
||||
result = enumerate_candidates(root, _region(function="empty"), "injection", taxonomy=load_families())
|
||||
assert result.candidates == () and result.reason == "no_candidate"
|
||||
|
||||
|
||||
def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates
|
||||
|
||||
# one call per line, so the arithmetic is the bound and not the fixture
|
||||
body = "\n".join(f" sink_{index}(value)" for index in range(MAX_CANDIDATES_PER_CALL + 5))
|
||||
root = _write(tmp_path, "app.py", f"def many(value):\n{body}\n")
|
||||
result = enumerate_candidates(root, _region(function="many"), "injection", taxonomy=load_families())
|
||||
assert len(result.candidates) == MAX_CANDIDATES_PER_CALL
|
||||
assert result.discarded == 5
|
||||
|
||||
|
||||
def test_enumerating_twice_returns_an_identical_sequence(tmp_path: Path) -> None:
|
||||
"""The batches, and therefore the sequence of model calls, are determined by the input alone (Req 2.4)."""
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", APP)
|
||||
taxonomy = load_families()
|
||||
first = enumerate_candidates(root, _region(function="constrained"), "injection", taxonomy=taxonomy)
|
||||
second = enumerate_candidates(root, _region(function="constrained"), "injection", taxonomy=taxonomy)
|
||||
assert [item.id for item in first.candidates] == [item.id for item in second.candidates]
|
||||
assert [item.id for item in first.candidates] == sorted({item.id for item in first.candidates}, key=_order(first))
|
||||
|
||||
|
||||
def _order(result): # type: ignore[no-untyped-def]
|
||||
index = {item.id: (item.path, item.line, item.name) for item in result.candidates}
|
||||
return lambda identifier: index[identifier]
|
||||
|
||||
|
||||
def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", APP)
|
||||
whole = enumerate_candidates(root, _region(function=None), "injection", taxonomy=load_families())
|
||||
named = enumerate_candidates(root, _region(function="leaky"), "injection", taxonomy=load_families())
|
||||
assert len(whole.candidates) >= len(named.candidates)
|
||||
assert {item.function for item in whole.candidates} > {"leaky"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("family", ["injection", "path", "deserialization", "untrusted_destination", "config_secrets"])
|
||||
def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path: Path, family: str) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", INJECTION)
|
||||
result = enumerate_candidates(root, _region(), family, taxonomy=load_families())
|
||||
assert result.reason in {"", "no_candidate"}
|
||||
if family in {"injection", "path", "deserialization", "untrusted_destination"}:
|
||||
assert result.candidates, f"{family} takes call sites and this file has three"
|
||||
Reference in New Issue
Block a user