mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
learn: in-function def-use slice from the tree-sitter layer + leak audit (\$0)
slice.py renders source -> step -> sink lines with fixed wording for flow, no-flow and not-analysable; the prompt option is default-off (byte-identical default prompt). semantic IR: JS/C declarations count as assignments, augmented assignment keeps its target (gate, map_gate, pair_gate outputs unchanged). Audit over exp-004's 501 units: 59 flow, 161 no-flow, 281 not analysable; has-flow within-pair accuracy 0.52, pooled AUC 0.55 -- no leak. Fixes a numbering bug that rejected every excerpt with a blank line. With Codex. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2836b78819
commit
1b2c7993a7
@@ -0,0 +1,175 @@
|
||||
"""Read-only, zero-model slice audit. Run: env -u OPENROUTER_API_KEY python benchmarks/learn/slice_leak_audit.py.
|
||||
|
||||
Only sources.toml itself is read, never its referenced corpora. Stored source IDs
|
||||
are the reporting strata (the store merges advisory pair catalogs into `pairs`).
|
||||
Ties get half credit. Both signed and best-direction scores are recorded; like
|
||||
learn.leaks, a reversed label shortcut is still a leak. Missing classes/pairs
|
||||
produce null metrics, not plausible zeros. Unsupported slices score zero for both
|
||||
indicators and remain separately counted in coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openultrasast.config import load_dotenv
|
||||
from openultrasast.learn.examples import load_examples
|
||||
from openultrasast.learn.labels import load_sources
|
||||
from openultrasast.learn.slice import function_slice
|
||||
from openultrasast.plane.memory import MemoryStore, open_store
|
||||
|
||||
UNITS = Path("plane/experiments/exp-004-pooled-block-gate.units.jsonl")
|
||||
OUTPUT = Path("benchmarks/measurements/2026-10-03-graph-slice-leak-audit/record.json")
|
||||
THRESHOLD = 0.70
|
||||
|
||||
|
||||
def complete_pairs(rows: Sequence[Mapping[str, Any]]) -> list[tuple[Mapping[str, Any], Mapping[str, Any]]]:
|
||||
groups: dict[tuple[str, str], list[Mapping[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
if row.get("pair"):
|
||||
groups[(row["family"], row["pair"])].append(row)
|
||||
pairs = []
|
||||
for members in groups.values():
|
||||
if len(members) == 2 and {m["label"] for m in members} == {0, 1}:
|
||||
v, f = sorted(members, key=lambda m: -m["label"])
|
||||
if v["group"] != f["group"] or v["fold"] != f["fold"]:
|
||||
raise ValueError("Pair crosses group or fold")
|
||||
pairs.append((v, f))
|
||||
elif len(members) != 1:
|
||||
raise ValueError("Ambiguous pair")
|
||||
return pairs
|
||||
|
||||
|
||||
def metrics(rows: Sequence[Mapping[str, Any]], signal: str) -> dict[str, Any]:
|
||||
pairs = complete_pairs(rows)
|
||||
higher = sum(v[signal] > f[signal] for v, f in pairs)
|
||||
lower = sum(v[signal] < f[signal] for v, f in pairs)
|
||||
ties = len(pairs) - higher - lower
|
||||
pair_score = (higher + 0.5 * ties) / len(pairs) if pairs else None
|
||||
positive = [r[signal] for r in rows if r["label"] == 1]
|
||||
negative = [r[signal] for r in rows if r["label"] == 0]
|
||||
auc = (
|
||||
sum((v > f) + 0.5 * (v == f) for v in positive for f in negative) / (len(positive) * len(negative))
|
||||
if positive and negative
|
||||
else None
|
||||
)
|
||||
best_pair = max(pair_score, 1 - pair_score) if pair_score is not None else None
|
||||
best_auc = max(auc, 1 - auc) if auc is not None else None
|
||||
return {
|
||||
"rows": len(rows),
|
||||
"pairs": len(pairs),
|
||||
"positive": len(positive),
|
||||
"negative": len(negative),
|
||||
"higher_on_vulnerable": higher,
|
||||
"higher_on_fixed": lower,
|
||||
"ties": ties,
|
||||
"within_pair_accuracy": pair_score,
|
||||
"pooled_auc": auc,
|
||||
"best_direction_within_pair_accuracy": best_pair,
|
||||
"best_direction_pooled_auc": best_auc,
|
||||
"flagged": any(v is not None and v >= THRESHOLD for v in (best_pair, best_auc)),
|
||||
}
|
||||
|
||||
|
||||
def audit(store: MemoryStore, units: Sequence[Mapping[str, Any]], *, expected_rows: int = 501, expected_pairs: int = 195) -> dict[str, Any]:
|
||||
if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
|
||||
raise ValueError(f"Expected {expected_rows} unique units, read {len(units)}")
|
||||
if len(complete_pairs(units)) != expected_pairs:
|
||||
raise ValueError(f"Expected {expected_pairs} complete pairs")
|
||||
examples = load_examples(store)
|
||||
index = {e.id: e for e in examples}
|
||||
if len(index) != len(examples):
|
||||
raise ValueError("Duplicate example IDs in memory")
|
||||
sources = {s.id for s in load_sources().sources}
|
||||
coverage: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
rows = []
|
||||
excerpt_bytes = 0
|
||||
blobs: dict[str, str] = {}
|
||||
for unit in units:
|
||||
e = index[unit["unit"]] # missing input is fatal, never counted as no-flow
|
||||
if (e.family, e.label, e.group) != (unit["family"], unit["label"], unit["group"]):
|
||||
raise ValueError(f"Unit metadata differs from store: {e.id}")
|
||||
if e.source not in sources:
|
||||
raise ValueError(f"Source absent from sources.toml: {e.source}")
|
||||
if e.excerpt_sha not in blobs:
|
||||
# MemoryStore.get_blob(verify=True) DELETES corrupt blobs: do not use it.
|
||||
data = store.get_blob("excerpts", e.excerpt_sha, verify=False)
|
||||
if not data or hashlib.sha256(data).hexdigest() != e.excerpt_sha:
|
||||
raise ValueError(f"Missing, empty or corrupt excerpt: {e.id}")
|
||||
excerpt_bytes += len(data)
|
||||
blobs[e.excerpt_sha] = data.decode("utf-8")
|
||||
result = function_slice(blobs[e.excerpt_sha], e.language, e.family)
|
||||
status = "not-analysable" if result is None else "flow" if result.steps else "no-flow"
|
||||
coverage[e.family][status] += 1
|
||||
rows.append(
|
||||
{
|
||||
**unit,
|
||||
"source": e.source,
|
||||
"status": status,
|
||||
"has_flow": int(status == "flow"),
|
||||
"slice_length": len(result.steps) if result else 0,
|
||||
}
|
||||
)
|
||||
pairs = complete_pairs(rows)
|
||||
asymmetry = Counter(
|
||||
"both" if v["has_flow"] and f["has_flow"] else "vulnerable_only" if v["has_flow"] else "fixed_only" if f["has_flow"] else "neither"
|
||||
for v, f in pairs
|
||||
)
|
||||
scopes = {"all": rows, **{f"source:{s}": [r for r in rows if r["source"] == s] for s in sorted({r["source"] for r in rows})}}
|
||||
return {
|
||||
"cost_usd": 0,
|
||||
"threshold": THRESHOLD,
|
||||
"unit_rows_read": len(units),
|
||||
"example_rows_read": len(examples),
|
||||
"matched_rows": len(rows),
|
||||
"excerpts_read": len(blobs),
|
||||
"excerpt_units": len(rows),
|
||||
"excerpt_bytes": excerpt_bytes,
|
||||
"complete_pairs": len(pairs),
|
||||
"coverage": {f: {s: c[s] for s in ("flow", "no-flow", "not-analysable")} for f, c in sorted(coverage.items())},
|
||||
"pair_asymmetry": {k: asymmetry[k] for k in ("vulnerable_only", "fixed_only", "both", "neither")},
|
||||
"signals": {
|
||||
signal: {scope: metrics(members, signal) for scope, members in scopes.items()} for signal in ("has_flow", "slice_length")
|
||||
},
|
||||
"source_strata": "Stored source IDs from sources.toml; advisory pair catalogs are merged into pairs by the example builder.",
|
||||
"limitations": (
|
||||
"Flat IR: no dominance/guards, textual branch order, potential parameter inputs, conservative call returns; no alias analysis."
|
||||
),
|
||||
"units": rows,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv(Path(".env"))
|
||||
os.environ.pop("OPENROUTER_API_KEY", None) # dotenv must not restore the removed key
|
||||
raw = UNITS.read_bytes()
|
||||
units = [json.loads(line) for line in raw.decode().splitlines() if line.strip()]
|
||||
report = audit(open_store(), units)
|
||||
report["units_bytes"] = len(raw)
|
||||
report["units_sha256"] = hashlib.sha256(raw).hexdigest()
|
||||
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUTPUT.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
totals = Counter({s: sum(c[s] for c in report["coverage"].values()) for s in ("flow", "no-flow", "not-analysable")})
|
||||
lines = [
|
||||
f"Slice leak audit: $0; units bytes={len(raw)}",
|
||||
f"Rows read: units={len(units)}, examples={report['example_rows_read']}, matched={report['matched_rows']}",
|
||||
f"Excerpts read: {report['excerpts_read']}; units={report['excerpt_units']}; bytes={report['excerpt_bytes']}",
|
||||
f"Coverage: {dict(totals)}",
|
||||
f"Complete pairs: {report['complete_pairs']}",
|
||||
f"Pair asymmetry: {report['pair_asymmetry']}",
|
||||
f"Has flow: {report['signals']['has_flow']['all']}",
|
||||
f"Slice length: {report['signals']['slice_length']['all']}",
|
||||
f"Source strata: {', '.join(report['signals']['has_flow'])}; threshold={THRESHOLD}",
|
||||
f"Report: {OUTPUT}",
|
||||
]
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -204,6 +204,7 @@ class ProgramSpec:
|
||||
priors: str = "off"
|
||||
inputs: str = DEFAULT_INPUTS # the input profile: which instruments the classifier never sees (v1: the engine)
|
||||
max_output_tokens: int = MAX_OUTPUT_TOKENS
|
||||
slice: bool = False # reserved experiment switch; rendering stays disabled until the leak audit is reviewed
|
||||
sampling: str = SAMPLING[0] # temperature | retrieval_ensemble (an experiment arm's setting; never compiled)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Bounded, local def-use evidence, never a vulnerability verdict.
|
||||
|
||||
Uses the shipped FileIR and fact/taxonomy routing. Parameters are potential inputs,
|
||||
as in semantic.taint; unknown call returns carry their arguments conservatively.
|
||||
Sanitizers stay on paths (they do not erase connectivity). Facts currently scope
|
||||
sanitizers by language, not family, so a sanitizer is not a discharge claim.
|
||||
The flat IR has no branch edges or dominance: no guard is asserted, and branch
|
||||
assignments are visited in textual order. No interprocedural or alias analysis.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from textwrap import dedent
|
||||
from typing import Literal
|
||||
|
||||
from ..model.specs import taint_specs
|
||||
from ..semantic.facts import load_facts
|
||||
from ..semantic.ir import Bind, CallSite, FunctionIR, parse_file
|
||||
from ..semantic.taint import _call_matches, _sanitizer_for, _source_in
|
||||
from .excerpt import redact
|
||||
|
||||
Role = Literal["source", "step", "sanitizer", "guard", "sink"]
|
||||
# The renderer uses two spaces; normalization strips the suffix on blank lines.
|
||||
_NUMBERED = re.compile(r"^\s*(\d+)(?: (.*))?$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True)
|
||||
class Step:
|
||||
line: int
|
||||
role: Role
|
||||
code: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Slice:
|
||||
steps: tuple[Step, ...] = ()
|
||||
|
||||
|
||||
def function_slice(excerpt_text: str, language: str, family: str) -> Slice | None:
|
||||
"""Return local paths, an empty slice, or None for an unsupported/unparseable excerpt.
|
||||
|
||||
Numbered excerpts retain their displayed numbers; raw code uses 1-based lines.
|
||||
Diffs are never parsed. A truncated/malformed excerpt cannot establish no-flow.
|
||||
Nested functions are analysed separately and never share definitions.
|
||||
"""
|
||||
raw = excerpt_text.split("\ndiff:\n", 1)[0].splitlines()
|
||||
matches = [_NUMBERED.match(line) for line in raw]
|
||||
if any(matches) and not all(matches):
|
||||
return None
|
||||
numbers = [int(m[1]) if m else i + 1 for i, m in enumerate(matches)]
|
||||
lines = [(m[2] or "") if m else line for m, line in zip(matches, raw, strict=True)]
|
||||
if len(set(numbers)) != len(numbers):
|
||||
return None
|
||||
code = dedent("\n".join(lines))
|
||||
ir = parse_file("excerpt", code, language)
|
||||
if not ir.parse_ok:
|
||||
return None
|
||||
spec = taint_specs(language=language).get(family)
|
||||
if spec is None:
|
||||
return Slice()
|
||||
shown = redact(lines, language)
|
||||
facts = load_facts().for_language(language)
|
||||
|
||||
def step(line: int, role: Role) -> Step:
|
||||
return Step(numbers[line - 1], role, shown[line - 1].strip())
|
||||
|
||||
def paths(function: FunctionIR) -> set[Step]:
|
||||
states = {name: {step(function.start_line, "source")} for name in function.params}
|
||||
found: set[Step] = set()
|
||||
|
||||
def value(text: str, names: tuple[str, ...], constant: bool, line: int, end: int) -> set[Step]:
|
||||
if constant:
|
||||
return set()
|
||||
result = set().union(*(states.get(name, set()) for name in names))
|
||||
if _source_in(text, facts):
|
||||
result.add(step(line, "source"))
|
||||
if result:
|
||||
# Keep only sanitizers whose call is inside this expression, not
|
||||
# an unrelated call elsewhere on the same source line.
|
||||
for call in function.calls:
|
||||
if (
|
||||
call.line >= line
|
||||
and (not end or call.order <= end)
|
||||
and any(
|
||||
re.search(r"(?<![\w$])" + re.escape(name) + r"\s*\(", text) and _call_matches(call.name, name)
|
||||
for name in spec.sanitizers
|
||||
)
|
||||
):
|
||||
result.add(step(call.line, "sanitizer"))
|
||||
return result
|
||||
|
||||
events: list[Bind | CallSite] = [*function.binds, *function.calls]
|
||||
# Tree-sitter supplies evaluation ends, including same-line statements.
|
||||
# The AST fallback supplies equivalent end-line/column positions.
|
||||
events.sort(key=lambda e: (e.order if e.order else e.line, isinstance(e, Bind)))
|
||||
for event in events:
|
||||
if isinstance(event, Bind):
|
||||
inherited = value(event.value_text, event.names, event.is_constant, event.line, event.order)
|
||||
states[event.name] = inherited | {step(event.line, "step")} if inherited else set()
|
||||
continue
|
||||
if not any(_call_matches(event.name, name) for name in spec.sinks):
|
||||
continue
|
||||
reached: set[Step] = set()
|
||||
for text, names, constant in zip(event.arg_texts, event.arg_names, event.arg_is_constant, strict=True):
|
||||
reached.update(value(text, names, constant, event.line, event.order))
|
||||
if reached:
|
||||
if _sanitizer_for(event, facts):
|
||||
reached.add(step(event.line, "sanitizer"))
|
||||
found.update(reached | {step(event.line, "sink")})
|
||||
return found
|
||||
|
||||
result: set[Step] = set()
|
||||
for function in ir.functions:
|
||||
result.update(paths(function))
|
||||
order = {"source": 0, "step": 1, "sanitizer": 2, "guard": 3, "sink": 4}
|
||||
return Slice(tuple(sorted(result, key=lambda s: (s.line, order[s.role]))))
|
||||
|
||||
|
||||
def render_slice(slice_or_none: Slice | None, reason: str) -> str:
|
||||
"""Fixed framing; reason is the language, never free-form provenance text."""
|
||||
prefix = "Data flow inside the function:"
|
||||
if slice_or_none is None:
|
||||
language = (
|
||||
reason
|
||||
if reason
|
||||
in {
|
||||
"python",
|
||||
"javascript",
|
||||
"typescript",
|
||||
"tsx",
|
||||
"c",
|
||||
"cpp",
|
||||
"java",
|
||||
"php",
|
||||
"go",
|
||||
"ruby",
|
||||
"rust",
|
||||
"csharp",
|
||||
"perl",
|
||||
"kotlin",
|
||||
"swift",
|
||||
"scala",
|
||||
"lua",
|
||||
"groovy",
|
||||
"shell",
|
||||
"other",
|
||||
}
|
||||
else "other"
|
||||
)
|
||||
return f"{prefix} not analysed ({language} is not supported)."
|
||||
if not slice_or_none.steps:
|
||||
return f"{prefix} no source reaches a sink of this family."
|
||||
return prefix + "\n" + "\n".join(f"L{s.line} {s.role}: {s.code}" for s in slice_or_none.steps)
|
||||
@@ -17,7 +17,7 @@ _KIND = {
|
||||
},
|
||||
"javascript": {
|
||||
"function": frozenset({"function_declaration", "method_definition", "arrow_function", "function_expression"}),
|
||||
"assign": frozenset({"assignment_expression", "augmented_assignment_expression"}),
|
||||
"assign": frozenset({"assignment_expression", "augmented_assignment_expression", "variable_declarator"}),
|
||||
"call": frozenset({"call_expression"}),
|
||||
"args": frozenset({"arguments"}),
|
||||
"string": frozenset({"string"}),
|
||||
@@ -25,7 +25,7 @@ _KIND = {
|
||||
},
|
||||
"c": {
|
||||
"function": frozenset({"function_definition"}),
|
||||
"assign": frozenset({"assignment_expression"}),
|
||||
"assign": frozenset({"assignment_expression", "init_declarator"}),
|
||||
"call": frozenset({"call_expression"}),
|
||||
"args": frozenset({"argument_list"}),
|
||||
"string": frozenset({"string_literal"}),
|
||||
@@ -152,12 +152,17 @@ def _bind(node: Any, kinds: dict[str, frozenset[str]]) -> Bind | None:
|
||||
if not name:
|
||||
return None
|
||||
value_text = _text(value_node) if value_node is not None else ""
|
||||
augmented = node.type in {"augmented_assignment", "augmented_assignment_expression"}
|
||||
names = _identifiers(value_node) if value_node is not None else ()
|
||||
if augmented:
|
||||
names = (name, *names)
|
||||
return Bind(
|
||||
name=name,
|
||||
order=int(node.end_byte),
|
||||
line=_line(node),
|
||||
value_text=value_text,
|
||||
is_constant=_is_constant(value_node, kinds) if value_node is not None else False,
|
||||
names=_identifiers(value_node) if value_node is not None else (),
|
||||
is_constant=not augmented and _is_constant(value_node, kinds) if value_node is not None else False,
|
||||
names=names,
|
||||
call_name=_call_name(value_node, kinds) if value_node is not None and value_node.type in kinds["call"] else None,
|
||||
)
|
||||
|
||||
@@ -167,6 +172,7 @@ def _call(node: Any, kinds: dict[str, frozenset[str]]) -> CallSite:
|
||||
arg_nodes = list(getattr(args_node, "named_children", ()) or ()) if args_node is not None else []
|
||||
extra = len(arg_nodes) >= 2 and arg_nodes[1].type in _SEQUENCE
|
||||
return CallSite(
|
||||
order=int(node.end_byte),
|
||||
name=_callee_name(node, args_node),
|
||||
line=_line(node),
|
||||
arg_texts=tuple(_text(arg) for arg in arg_nodes),
|
||||
@@ -211,6 +217,9 @@ def _function_name(node: Any) -> str:
|
||||
|
||||
|
||||
def _params(node: Any, kinds: dict[str, frozenset[str]]) -> tuple[str, ...]:
|
||||
parameter = node.child_by_field_name("parameter")
|
||||
if parameter is not None:
|
||||
return _identifiers(parameter)
|
||||
for child in _walk(node, skip=frozenset()):
|
||||
if child.type in kinds["params"]:
|
||||
return _identifiers(child)
|
||||
|
||||
@@ -15,6 +15,7 @@ class Bind:
|
||||
is_constant: bool
|
||||
names: tuple[str, ...]
|
||||
call_name: str | None
|
||||
order: int = 0 # evaluation end position; calls precede their containing assignment
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -25,6 +26,7 @@ class CallSite:
|
||||
arg_is_constant: tuple[bool, ...]
|
||||
arg_names: tuple[tuple[str, ...], ...]
|
||||
extra_arg_is_sequence: bool
|
||||
order: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -119,6 +121,7 @@ def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> N
|
||||
is_constant=value_bind[1],
|
||||
names=value_bind[2],
|
||||
call_name=value_bind[3],
|
||||
order=_end_position(stmt.value),
|
||||
)
|
||||
)
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
@@ -133,6 +136,7 @@ def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> N
|
||||
is_constant=value_bind[1],
|
||||
names=value_bind[2],
|
||||
call_name=value_bind[3],
|
||||
order=_end_position(stmt.value),
|
||||
)
|
||||
)
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
@@ -147,6 +151,7 @@ def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> N
|
||||
is_constant=False,
|
||||
names=tuple(sorted(set(value_bind[2]) | {stmt.target.id})),
|
||||
call_name=value_bind[3],
|
||||
order=_end_position(stmt.value),
|
||||
)
|
||||
)
|
||||
_collect_expr_calls(stmt.value, calls)
|
||||
@@ -190,6 +195,7 @@ def _call_site(node: ast.Call) -> CallSite:
|
||||
if len(node.args) >= 2:
|
||||
extra_sequence = isinstance(node.args[1], ast.Tuple | ast.List | ast.Set)
|
||||
return CallSite(
|
||||
order=_end_position(node),
|
||||
name=_call_name(node),
|
||||
line=node.lineno,
|
||||
arg_texts=tuple(arg_texts),
|
||||
@@ -199,6 +205,10 @@ def _call_site(node: ast.Call) -> CallSite:
|
||||
)
|
||||
|
||||
|
||||
def _end_position(node: ast.AST) -> int:
|
||||
return int(getattr(node, "end_lineno", 0) or 0) * 1_000_000 + int(getattr(node, "end_col_offset", 0) or 0)
|
||||
|
||||
|
||||
def _call_name(node: ast.AST) -> str:
|
||||
func = node.func if isinstance(node, ast.Call) else node
|
||||
return _unparse(func)
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Local slices and zero-cost audit: authored examples only, no memory/network/model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learn.slice import Slice, function_slice, render_slice
|
||||
|
||||
|
||||
def sliced(code: str, language: str = "python", family: str = "injection") -> Slice:
|
||||
result = function_slice(code, language, family)
|
||||
assert result is not None
|
||||
return result
|
||||
|
||||
|
||||
def test_learn_slice_def_use_preserves_excerpt_lines_and_ignores_diff() -> None:
|
||||
result = sliced(
|
||||
' 40 def f():\n 41 x = request.args["x"]\n 42 y = wrap(x)\n 43 execute(y)\ndiff:\n+execute(other)'
|
||||
)
|
||||
assert [(s.line, s.role) for s in result.steps] == [(41, "source"), (41, "step"), (42, "step"), (43, "sink")]
|
||||
assert all(40 <= s.line <= 43 for s in result.steps)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blank", ["", " ", "\t"])
|
||||
@pytest.mark.parametrize("indent", [" ", "\t"])
|
||||
def test_learn_slice_numbered_blank_lines_preserve_displayed_lines(blank: str, indent: str) -> None:
|
||||
from openultrasast.learn.excerpt import excerpt
|
||||
|
||||
lines = [""] * 10 + ["def f(x):", blank, f"{indent}y = x", blank, f"{indent}execute(y)"]
|
||||
rendered = excerpt(lines, "python", "f")
|
||||
assert rendered is not None
|
||||
assert rendered.text.splitlines()[1] == " 12"
|
||||
assert rendered.text.splitlines()[3] == " 14"
|
||||
result = sliced(rendered.text)
|
||||
assert [(s.line, s.role) for s in result.steps] == [(11, "source"), (13, "step"), (15, "sink")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mixed_line", [" execute(x)", " 12 execute(x)", " 12\texecute(x)"])
|
||||
def test_learn_slice_mixed_numbering_is_not_analysed(mixed_line: str) -> None:
|
||||
assert function_slice(f" 10 def f(x):\n 11\n{mixed_line}", "python", "injection") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"code",
|
||||
[
|
||||
'def f():\n execute("request.args")',
|
||||
'def f():\n execute(x)\n x = request.args["x"]',
|
||||
'def f(x):\n x = "safe"\n execute(x)',
|
||||
'def f():\n x = request.args["x"]\n execute("safe")',
|
||||
'def f():\n x = request.args["x"]\n def inner():\n execute(x)',
|
||||
],
|
||||
)
|
||||
def test_learn_slice_no_invented_def_use(code: str) -> None:
|
||||
assert sliced(code).steps == ()
|
||||
|
||||
|
||||
def test_learn_slice_parameters_and_nested_returns() -> None:
|
||||
assert [(s.line, s.role) for s in sliced("def f(x):\n return execute(transform(x))").steps] == [(1, "source"), (2, "sink")]
|
||||
|
||||
|
||||
def test_learn_slice_sanitizer_keeps_connectivity_and_does_not_claim_guard() -> None:
|
||||
result = sliced("def f(x):\n y = ast.literal_eval(x)\n execute(y)\n ast.literal_eval(other)")
|
||||
assert (2, "sanitizer") in [(s.line, s.role) for s in result.steps]
|
||||
assert not any(s.line == 4 for s in result.steps)
|
||||
checked = sliced("def f(x):\n if is_safe_url(x):\n redirect(x)", family="untrusted_destination")
|
||||
assert not any(s.role == "guard" for s in checked.steps) # flat IR cannot prove dominance
|
||||
|
||||
|
||||
def test_learn_slice_parameterized_sink_and_family_routing() -> None:
|
||||
result = sliced('def f(x):\n execute("select ?", [x])')
|
||||
assert {s.role for s in result.steps} == {"source", "sanitizer", "sink"}
|
||||
assert sliced("def f(x):\n execute(x)", family="path").steps == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"language,code",
|
||||
[
|
||||
("javascript", "function f(req) { const x = req.query.x; const y = x; eval(y); }"),
|
||||
("typescript", "function f(req: any) { const x = req.query.x; eval(x); }"),
|
||||
("c", "void f(char *x) { char *y = x; system(y); }"),
|
||||
("java", "class C { void f(String x) { String y = x; exec(y); } }"),
|
||||
],
|
||||
)
|
||||
def test_learn_slice_shared_cst_declarations(language: str, code: str) -> None:
|
||||
assert any(s.role == "sink" for s in sliced(code, language).steps)
|
||||
|
||||
|
||||
def test_learn_slice_same_line_assignment_order() -> None:
|
||||
assert not sliced('function f(x) { x = "safe"; eval(x); }', "javascript").steps
|
||||
assert sliced('function f(x) { eval(x); x = "safe"; }', "javascript").steps
|
||||
assert sliced("function f() { const x = req.query.x; eval(x); }", "javascript").steps
|
||||
|
||||
|
||||
def test_learn_slice_fixed_framing_redaction_and_unsupported() -> None:
|
||||
assert function_slice("<?php function f($x) { exec($x); }", "php", "injection") is None
|
||||
assert render_slice(None, "php") == "Data flow inside the function: not analysed (php is not supported)."
|
||||
assert render_slice(Slice(), "anything") == "Data flow inside the function: no source reaches a sink of this family."
|
||||
assert "CVE" not in render_slice(None, "CVE-2026-1234")
|
||||
result = sliced("def f(x):\n execute(x) # CVE-2026-1234 vulnerable")
|
||||
shown = render_slice(result, "python")
|
||||
assert shown.startswith("Data flow inside the function:\nL1 source:")
|
||||
assert "CVE-2026-1234" not in shown and "vulnerable" not in shown
|
||||
assert function_slice("def f(x):\n execute(", "python", "injection") is None
|
||||
|
||||
|
||||
def test_learn_slice_default_prompt_frozen_before_change() -> None:
|
||||
import test_learn_program as baseline
|
||||
|
||||
prompt, _ = baseline.program().prepare(baseline.candidate(), baseline.FOLD)
|
||||
assert hashlib.sha256(json.dumps(prompt.messages, sort_keys=True).encode()).hexdigest() == (
|
||||
"f6b5cfb5a8b7cb1591d7b4534e254ad2a349452a03e825bdcf250e5f60b4e80b"
|
||||
)
|
||||
assert baseline.ProgramSpec().slice is False
|
||||
reserved, _ = baseline.program(slice=True).prepare(baseline.candidate(), baseline.FOLD)
|
||||
assert reserved.messages == prompt.messages # no prompt wiring before audit review
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audit_module() -> ModuleType:
|
||||
path = Path(__file__).resolve().parents[1] / "benchmarks/learn/slice_leak_audit.py"
|
||||
spec = importlib.util.spec_from_file_location("slice_leak_audit", path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_learn_slice_audit_metrics_ties_and_reverse_leak(audit_module: ModuleType) -> None:
|
||||
rows = [
|
||||
{"family": "injection", "group": "g", "fold": "f", "pair": str(pair), "label": label, "has_flow": 1 - label, "slice_length": 3}
|
||||
for pair in range(10)
|
||||
for label in (0, 1)
|
||||
]
|
||||
metric = audit_module.metrics(rows, "has_flow")
|
||||
assert metric["within_pair_accuracy"] == metric["pooled_auc"] == 0
|
||||
assert metric["flagged"] and metric["best_direction_pooled_auc"] == 1
|
||||
metric = audit_module.metrics(rows, "slice_length")
|
||||
assert metric["within_pair_accuracy"] == metric["pooled_auc"] == 0.5
|
||||
assert not metric["flagged"]
|
||||
assert audit_module.metrics([], "has_flow")["pooled_auc"] is None
|
||||
|
||||
|
||||
def test_learn_slice_audit_reads_and_never_writes(audit_module: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from learn_fixtures import corpus
|
||||
|
||||
raw = b" 10 def f(x):\n 11 execute(x)\n"
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
example = replace(corpus(groups=1)[0], excerpt_sha=digest, source="pairs", language="python", family="injection")
|
||||
monkeypatch.setattr(audit_module, "load_examples", lambda store: [example])
|
||||
|
||||
class ReadOnly:
|
||||
def get_blob(self, prefix: str, name: str, *, verify: bool = True) -> bytes:
|
||||
assert (prefix, name, verify) == ("excerpts", digest, False)
|
||||
return raw
|
||||
|
||||
units = [{"family": example.family, "label": example.label, "group": example.group, "unit": example.id, "pair": None, "fold": "f"}]
|
||||
report = audit_module.audit(ReadOnly(), units, expected_rows=1, expected_pairs=0)
|
||||
assert report["example_rows_read"] == report["matched_rows"] == report["excerpts_read"] == 1
|
||||
assert report["excerpt_bytes"] == len(raw)
|
||||
assert report["coverage"]["injection"]["flow"] == 1
|
||||
|
||||
class Corrupt(ReadOnly):
|
||||
def get_blob(self, prefix: str, name: str, *, verify: bool = True) -> bytes:
|
||||
assert not verify
|
||||
return b"corrupt"
|
||||
|
||||
with pytest.raises(ValueError, match="corrupt excerpt"):
|
||||
audit_module.audit(Corrupt(), units, expected_rows=1, expected_pairs=0)
|
||||
|
||||
|
||||
def test_learn_slice_augmented_assignment_preserves_value() -> None:
|
||||
result = sliced('def f(x):\n x += " suffix"\n execute(x)')
|
||||
assert {s.line for s in result.steps} == {1, 2, 3}
|
||||
result = sliced('function f(x) { x += " suffix"; eval(x); }', "javascript")
|
||||
assert any(s.role == "sink" for s in result.steps)
|
||||
|
||||
|
||||
def test_learn_slice_python_fallback_order(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.semantic import cst
|
||||
|
||||
monkeypatch.setattr(cst, "parse_with_cst", lambda *args: None)
|
||||
assert sliced("def f():\n x = input(); execute(x)").steps
|
||||
assert not sliced('def f(x):\n x = "safe"; execute(x)').steps
|
||||
|
||||
|
||||
def test_learn_slice_bare_arrow_parameter() -> None:
|
||||
assert sliced("const f = x => eval(x);", "javascript").steps
|
||||
Reference in New Issue
Block a user