learn: in-function def-use slice from the tree-sitter layer + leak audit (\$0)

slice.py renders source -> step -> sink lines with fixed wording for flow,
no-flow and not-analysable; the prompt option is default-off (byte-identical
default prompt). semantic IR: JS/C declarations count as assignments, augmented
assignment keeps its target (gate, map_gate, pair_gate outputs unchanged).
Audit over exp-004's 501 units: 59 flow, 161 no-flow, 281 not analysable;
has-flow within-pair accuracy 0.52, pooled AUC 0.55 -- no leak. Fixes a
numbering bug that rejected every excerpt with a blank line. With Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-03 14:56:52 +02:00
co-authored by Claude Opus 5.5
parent 2836b78819
commit 1b2c7993a7
7 changed files with 6789 additions and 4 deletions
+175
View File
@@ -0,0 +1,175 @@
"""Read-only, zero-model slice audit. Run: env -u OPENROUTER_API_KEY python benchmarks/learn/slice_leak_audit.py.
Only sources.toml itself is read, never its referenced corpora. Stored source IDs
are the reporting strata (the store merges advisory pair catalogs into `pairs`).
Ties get half credit. Both signed and best-direction scores are recorded; like
learn.leaks, a reversed label shortcut is still a leak. Missing classes/pairs
produce null metrics, not plausible zeros. Unsupported slices score zero for both
indicators and remain separately counted in coverage.
"""
from __future__ import annotations
import hashlib
import json
import os
from collections import Counter, defaultdict
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from openultrasast.config import load_dotenv
from openultrasast.learn.examples import load_examples
from openultrasast.learn.labels import load_sources
from openultrasast.learn.slice import function_slice
from openultrasast.plane.memory import MemoryStore, open_store
UNITS = Path("plane/experiments/exp-004-pooled-block-gate.units.jsonl")
OUTPUT = Path("benchmarks/measurements/2026-10-03-graph-slice-leak-audit/record.json")
THRESHOLD = 0.70
def complete_pairs(rows: Sequence[Mapping[str, Any]]) -> list[tuple[Mapping[str, Any], Mapping[str, Any]]]:
groups: dict[tuple[str, str], list[Mapping[str, Any]]] = defaultdict(list)
for row in rows:
if row.get("pair"):
groups[(row["family"], row["pair"])].append(row)
pairs = []
for members in groups.values():
if len(members) == 2 and {m["label"] for m in members} == {0, 1}:
v, f = sorted(members, key=lambda m: -m["label"])
if v["group"] != f["group"] or v["fold"] != f["fold"]:
raise ValueError("Pair crosses group or fold")
pairs.append((v, f))
elif len(members) != 1:
raise ValueError("Ambiguous pair")
return pairs
def metrics(rows: Sequence[Mapping[str, Any]], signal: str) -> dict[str, Any]:
pairs = complete_pairs(rows)
higher = sum(v[signal] > f[signal] for v, f in pairs)
lower = sum(v[signal] < f[signal] for v, f in pairs)
ties = len(pairs) - higher - lower
pair_score = (higher + 0.5 * ties) / len(pairs) if pairs else None
positive = [r[signal] for r in rows if r["label"] == 1]
negative = [r[signal] for r in rows if r["label"] == 0]
auc = (
sum((v > f) + 0.5 * (v == f) for v in positive for f in negative) / (len(positive) * len(negative))
if positive and negative
else None
)
best_pair = max(pair_score, 1 - pair_score) if pair_score is not None else None
best_auc = max(auc, 1 - auc) if auc is not None else None
return {
"rows": len(rows),
"pairs": len(pairs),
"positive": len(positive),
"negative": len(negative),
"higher_on_vulnerable": higher,
"higher_on_fixed": lower,
"ties": ties,
"within_pair_accuracy": pair_score,
"pooled_auc": auc,
"best_direction_within_pair_accuracy": best_pair,
"best_direction_pooled_auc": best_auc,
"flagged": any(v is not None and v >= THRESHOLD for v in (best_pair, best_auc)),
}
def audit(store: MemoryStore, units: Sequence[Mapping[str, Any]], *, expected_rows: int = 501, expected_pairs: int = 195) -> dict[str, Any]:
if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
raise ValueError(f"Expected {expected_rows} unique units, read {len(units)}")
if len(complete_pairs(units)) != expected_pairs:
raise ValueError(f"Expected {expected_pairs} complete pairs")
examples = load_examples(store)
index = {e.id: e for e in examples}
if len(index) != len(examples):
raise ValueError("Duplicate example IDs in memory")
sources = {s.id for s in load_sources().sources}
coverage: dict[str, Counter[str]] = defaultdict(Counter)
rows = []
excerpt_bytes = 0
blobs: dict[str, str] = {}
for unit in units:
e = index[unit["unit"]] # missing input is fatal, never counted as no-flow
if (e.family, e.label, e.group) != (unit["family"], unit["label"], unit["group"]):
raise ValueError(f"Unit metadata differs from store: {e.id}")
if e.source not in sources:
raise ValueError(f"Source absent from sources.toml: {e.source}")
if e.excerpt_sha not in blobs:
# MemoryStore.get_blob(verify=True) DELETES corrupt blobs: do not use it.
data = store.get_blob("excerpts", e.excerpt_sha, verify=False)
if not data or hashlib.sha256(data).hexdigest() != e.excerpt_sha:
raise ValueError(f"Missing, empty or corrupt excerpt: {e.id}")
excerpt_bytes += len(data)
blobs[e.excerpt_sha] = data.decode("utf-8")
result = function_slice(blobs[e.excerpt_sha], e.language, e.family)
status = "not-analysable" if result is None else "flow" if result.steps else "no-flow"
coverage[e.family][status] += 1
rows.append(
{
**unit,
"source": e.source,
"status": status,
"has_flow": int(status == "flow"),
"slice_length": len(result.steps) if result else 0,
}
)
pairs = complete_pairs(rows)
asymmetry = Counter(
"both" if v["has_flow"] and f["has_flow"] else "vulnerable_only" if v["has_flow"] else "fixed_only" if f["has_flow"] else "neither"
for v, f in pairs
)
scopes = {"all": rows, **{f"source:{s}": [r for r in rows if r["source"] == s] for s in sorted({r["source"] for r in rows})}}
return {
"cost_usd": 0,
"threshold": THRESHOLD,
"unit_rows_read": len(units),
"example_rows_read": len(examples),
"matched_rows": len(rows),
"excerpts_read": len(blobs),
"excerpt_units": len(rows),
"excerpt_bytes": excerpt_bytes,
"complete_pairs": len(pairs),
"coverage": {f: {s: c[s] for s in ("flow", "no-flow", "not-analysable")} for f, c in sorted(coverage.items())},
"pair_asymmetry": {k: asymmetry[k] for k in ("vulnerable_only", "fixed_only", "both", "neither")},
"signals": {
signal: {scope: metrics(members, signal) for scope, members in scopes.items()} for signal in ("has_flow", "slice_length")
},
"source_strata": "Stored source IDs from sources.toml; advisory pair catalogs are merged into pairs by the example builder.",
"limitations": (
"Flat IR: no dominance/guards, textual branch order, potential parameter inputs, conservative call returns; no alias analysis."
),
"units": rows,
}
def main() -> None:
load_dotenv(Path(".env"))
os.environ.pop("OPENROUTER_API_KEY", None) # dotenv must not restore the removed key
raw = UNITS.read_bytes()
units = [json.loads(line) for line in raw.decode().splitlines() if line.strip()]
report = audit(open_store(), units)
report["units_bytes"] = len(raw)
report["units_sha256"] = hashlib.sha256(raw).hexdigest()
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
OUTPUT.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
totals = Counter({s: sum(c[s] for c in report["coverage"].values()) for s in ("flow", "no-flow", "not-analysable")})
lines = [
f"Slice leak audit: $0; units bytes={len(raw)}",
f"Rows read: units={len(units)}, examples={report['example_rows_read']}, matched={report['matched_rows']}",
f"Excerpts read: {report['excerpts_read']}; units={report['excerpt_units']}; bytes={report['excerpt_bytes']}",
f"Coverage: {dict(totals)}",
f"Complete pairs: {report['complete_pairs']}",
f"Pair asymmetry: {report['pair_asymmetry']}",
f"Has flow: {report['signals']['has_flow']['all']}",
f"Slice length: {report['signals']['slice_length']['all']}",
f"Source strata: {', '.join(report['signals']['has_flow'])}; threshold={THRESHOLD}",
f"Report: {OUTPUT}",
]
print("\n".join(lines))
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
+1
View File
@@ -204,6 +204,7 @@ class ProgramSpec:
priors: str = "off"
inputs: str = DEFAULT_INPUTS # the input profile: which instruments the classifier never sees (v1: the engine)
max_output_tokens: int = MAX_OUTPUT_TOKENS
slice: bool = False # reserved experiment switch; rendering stays disabled until the leak audit is reviewed
sampling: str = SAMPLING[0] # temperature | retrieval_ensemble (an experiment arm's setting; never compiled)
+155
View File
@@ -0,0 +1,155 @@
"""Bounded, local def-use evidence, never a vulnerability verdict.
Uses the shipped FileIR and fact/taxonomy routing. Parameters are potential inputs,
as in semantic.taint; unknown call returns carry their arguments conservatively.
Sanitizers stay on paths (they do not erase connectivity). Facts currently scope
sanitizers by language, not family, so a sanitizer is not a discharge claim.
The flat IR has no branch edges or dominance: no guard is asserted, and branch
assignments are visited in textual order. No interprocedural or alias analysis.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from textwrap import dedent
from typing import Literal
from ..model.specs import taint_specs
from ..semantic.facts import load_facts
from ..semantic.ir import Bind, CallSite, FunctionIR, parse_file
from ..semantic.taint import _call_matches, _sanitizer_for, _source_in
from .excerpt import redact
Role = Literal["source", "step", "sanitizer", "guard", "sink"]
# The renderer uses two spaces; normalization strips the suffix on blank lines.
_NUMBERED = re.compile(r"^\s*(\d+)(?: (.*))?$")
@dataclass(frozen=True, order=True)
class Step:
line: int
role: Role
code: str
@dataclass(frozen=True)
class Slice:
steps: tuple[Step, ...] = ()
def function_slice(excerpt_text: str, language: str, family: str) -> Slice | None:
"""Return local paths, an empty slice, or None for an unsupported/unparseable excerpt.
Numbered excerpts retain their displayed numbers; raw code uses 1-based lines.
Diffs are never parsed. A truncated/malformed excerpt cannot establish no-flow.
Nested functions are analysed separately and never share definitions.
"""
raw = excerpt_text.split("\ndiff:\n", 1)[0].splitlines()
matches = [_NUMBERED.match(line) for line in raw]
if any(matches) and not all(matches):
return None
numbers = [int(m[1]) if m else i + 1 for i, m in enumerate(matches)]
lines = [(m[2] or "") if m else line for m, line in zip(matches, raw, strict=True)]
if len(set(numbers)) != len(numbers):
return None
code = dedent("\n".join(lines))
ir = parse_file("excerpt", code, language)
if not ir.parse_ok:
return None
spec = taint_specs(language=language).get(family)
if spec is None:
return Slice()
shown = redact(lines, language)
facts = load_facts().for_language(language)
def step(line: int, role: Role) -> Step:
return Step(numbers[line - 1], role, shown[line - 1].strip())
def paths(function: FunctionIR) -> set[Step]:
states = {name: {step(function.start_line, "source")} for name in function.params}
found: set[Step] = set()
def value(text: str, names: tuple[str, ...], constant: bool, line: int, end: int) -> set[Step]:
if constant:
return set()
result = set().union(*(states.get(name, set()) for name in names))
if _source_in(text, facts):
result.add(step(line, "source"))
if result:
# Keep only sanitizers whose call is inside this expression, not
# an unrelated call elsewhere on the same source line.
for call in function.calls:
if (
call.line >= line
and (not end or call.order <= end)
and any(
re.search(r"(?<![\w$])" + re.escape(name) + r"\s*\(", text) and _call_matches(call.name, name)
for name in spec.sanitizers
)
):
result.add(step(call.line, "sanitizer"))
return result
events: list[Bind | CallSite] = [*function.binds, *function.calls]
# Tree-sitter supplies evaluation ends, including same-line statements.
# The AST fallback supplies equivalent end-line/column positions.
events.sort(key=lambda e: (e.order if e.order else e.line, isinstance(e, Bind)))
for event in events:
if isinstance(event, Bind):
inherited = value(event.value_text, event.names, event.is_constant, event.line, event.order)
states[event.name] = inherited | {step(event.line, "step")} if inherited else set()
continue
if not any(_call_matches(event.name, name) for name in spec.sinks):
continue
reached: set[Step] = set()
for text, names, constant in zip(event.arg_texts, event.arg_names, event.arg_is_constant, strict=True):
reached.update(value(text, names, constant, event.line, event.order))
if reached:
if _sanitizer_for(event, facts):
reached.add(step(event.line, "sanitizer"))
found.update(reached | {step(event.line, "sink")})
return found
result: set[Step] = set()
for function in ir.functions:
result.update(paths(function))
order = {"source": 0, "step": 1, "sanitizer": 2, "guard": 3, "sink": 4}
return Slice(tuple(sorted(result, key=lambda s: (s.line, order[s.role]))))
def render_slice(slice_or_none: Slice | None, reason: str) -> str:
"""Fixed framing; reason is the language, never free-form provenance text."""
prefix = "Data flow inside the function:"
if slice_or_none is None:
language = (
reason
if reason
in {
"python",
"javascript",
"typescript",
"tsx",
"c",
"cpp",
"java",
"php",
"go",
"ruby",
"rust",
"csharp",
"perl",
"kotlin",
"swift",
"scala",
"lua",
"groovy",
"shell",
"other",
}
else "other"
)
return f"{prefix} not analysed ({language} is not supported)."
if not slice_or_none.steps:
return f"{prefix} no source reaches a sink of this family."
return prefix + "\n" + "\n".join(f"L{s.line} {s.role}: {s.code}" for s in slice_or_none.steps)
+13 -4
View File
@@ -17,7 +17,7 @@ _KIND = {
},
"javascript": {
"function": frozenset({"function_declaration", "method_definition", "arrow_function", "function_expression"}),
"assign": frozenset({"assignment_expression", "augmented_assignment_expression"}),
"assign": frozenset({"assignment_expression", "augmented_assignment_expression", "variable_declarator"}),
"call": frozenset({"call_expression"}),
"args": frozenset({"arguments"}),
"string": frozenset({"string"}),
@@ -25,7 +25,7 @@ _KIND = {
},
"c": {
"function": frozenset({"function_definition"}),
"assign": frozenset({"assignment_expression"}),
"assign": frozenset({"assignment_expression", "init_declarator"}),
"call": frozenset({"call_expression"}),
"args": frozenset({"argument_list"}),
"string": frozenset({"string_literal"}),
@@ -152,12 +152,17 @@ def _bind(node: Any, kinds: dict[str, frozenset[str]]) -> Bind | None:
if not name:
return None
value_text = _text(value_node) if value_node is not None else ""
augmented = node.type in {"augmented_assignment", "augmented_assignment_expression"}
names = _identifiers(value_node) if value_node is not None else ()
if augmented:
names = (name, *names)
return Bind(
name=name,
order=int(node.end_byte),
line=_line(node),
value_text=value_text,
is_constant=_is_constant(value_node, kinds) if value_node is not None else False,
names=_identifiers(value_node) if value_node is not None else (),
is_constant=not augmented and _is_constant(value_node, kinds) if value_node is not None else False,
names=names,
call_name=_call_name(value_node, kinds) if value_node is not None and value_node.type in kinds["call"] else None,
)
@@ -167,6 +172,7 @@ def _call(node: Any, kinds: dict[str, frozenset[str]]) -> CallSite:
arg_nodes = list(getattr(args_node, "named_children", ()) or ()) if args_node is not None else []
extra = len(arg_nodes) >= 2 and arg_nodes[1].type in _SEQUENCE
return CallSite(
order=int(node.end_byte),
name=_callee_name(node, args_node),
line=_line(node),
arg_texts=tuple(_text(arg) for arg in arg_nodes),
@@ -211,6 +217,9 @@ def _function_name(node: Any) -> str:
def _params(node: Any, kinds: dict[str, frozenset[str]]) -> tuple[str, ...]:
parameter = node.child_by_field_name("parameter")
if parameter is not None:
return _identifiers(parameter)
for child in _walk(node, skip=frozenset()):
if child.type in kinds["params"]:
return _identifiers(child)
+10
View File
@@ -15,6 +15,7 @@ class Bind:
is_constant: bool
names: tuple[str, ...]
call_name: str | None
order: int = 0 # evaluation end position; calls precede their containing assignment
@dataclass(frozen=True)
@@ -25,6 +26,7 @@ class CallSite:
arg_is_constant: tuple[bool, ...]
arg_names: tuple[tuple[str, ...], ...]
extra_arg_is_sequence: bool
order: int = 0
@dataclass(frozen=True)
@@ -119,6 +121,7 @@ def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> N
is_constant=value_bind[1],
names=value_bind[2],
call_name=value_bind[3],
order=_end_position(stmt.value),
)
)
_collect_expr_calls(stmt.value, calls)
@@ -133,6 +136,7 @@ def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> N
is_constant=value_bind[1],
names=value_bind[2],
call_name=value_bind[3],
order=_end_position(stmt.value),
)
)
_collect_expr_calls(stmt.value, calls)
@@ -147,6 +151,7 @@ def _collect_stmt(stmt: ast.stmt, binds: list[Bind], calls: list[CallSite]) -> N
is_constant=False,
names=tuple(sorted(set(value_bind[2]) | {stmt.target.id})),
call_name=value_bind[3],
order=_end_position(stmt.value),
)
)
_collect_expr_calls(stmt.value, calls)
@@ -190,6 +195,7 @@ def _call_site(node: ast.Call) -> CallSite:
if len(node.args) >= 2:
extra_sequence = isinstance(node.args[1], ast.Tuple | ast.List | ast.Set)
return CallSite(
order=_end_position(node),
name=_call_name(node),
line=node.lineno,
arg_texts=tuple(arg_texts),
@@ -199,6 +205,10 @@ def _call_site(node: ast.Call) -> CallSite:
)
def _end_position(node: ast.AST) -> int:
return int(getattr(node, "end_lineno", 0) or 0) * 1_000_000 + int(getattr(node, "end_col_offset", 0) or 0)
def _call_name(node: ast.AST) -> str:
func = node.func if isinstance(node, ast.Call) else node
return _unparse(func)
+194
View File
@@ -0,0 +1,194 @@
"""Local slices and zero-cost audit: authored examples only, no memory/network/model."""
from __future__ import annotations
import hashlib
import importlib.util
import json
from dataclasses import replace
from pathlib import Path
from types import ModuleType
import pytest
from openultrasast.learn.slice import Slice, function_slice, render_slice
def sliced(code: str, language: str = "python", family: str = "injection") -> Slice:
result = function_slice(code, language, family)
assert result is not None
return result
def test_learn_slice_def_use_preserves_excerpt_lines_and_ignores_diff() -> None:
result = sliced(
' 40 def f():\n 41 x = request.args["x"]\n 42 y = wrap(x)\n 43 execute(y)\ndiff:\n+execute(other)'
)
assert [(s.line, s.role) for s in result.steps] == [(41, "source"), (41, "step"), (42, "step"), (43, "sink")]
assert all(40 <= s.line <= 43 for s in result.steps)
@pytest.mark.parametrize("blank", ["", " ", "\t"])
@pytest.mark.parametrize("indent", [" ", "\t"])
def test_learn_slice_numbered_blank_lines_preserve_displayed_lines(blank: str, indent: str) -> None:
from openultrasast.learn.excerpt import excerpt
lines = [""] * 10 + ["def f(x):", blank, f"{indent}y = x", blank, f"{indent}execute(y)"]
rendered = excerpt(lines, "python", "f")
assert rendered is not None
assert rendered.text.splitlines()[1] == " 12"
assert rendered.text.splitlines()[3] == " 14"
result = sliced(rendered.text)
assert [(s.line, s.role) for s in result.steps] == [(11, "source"), (13, "step"), (15, "sink")]
@pytest.mark.parametrize("mixed_line", [" execute(x)", " 12 execute(x)", " 12\texecute(x)"])
def test_learn_slice_mixed_numbering_is_not_analysed(mixed_line: str) -> None:
assert function_slice(f" 10 def f(x):\n 11\n{mixed_line}", "python", "injection") is None
@pytest.mark.parametrize(
"code",
[
'def f():\n execute("request.args")',
'def f():\n execute(x)\n x = request.args["x"]',
'def f(x):\n x = "safe"\n execute(x)',
'def f():\n x = request.args["x"]\n execute("safe")',
'def f():\n x = request.args["x"]\n def inner():\n execute(x)',
],
)
def test_learn_slice_no_invented_def_use(code: str) -> None:
assert sliced(code).steps == ()
def test_learn_slice_parameters_and_nested_returns() -> None:
assert [(s.line, s.role) for s in sliced("def f(x):\n return execute(transform(x))").steps] == [(1, "source"), (2, "sink")]
def test_learn_slice_sanitizer_keeps_connectivity_and_does_not_claim_guard() -> None:
result = sliced("def f(x):\n y = ast.literal_eval(x)\n execute(y)\n ast.literal_eval(other)")
assert (2, "sanitizer") in [(s.line, s.role) for s in result.steps]
assert not any(s.line == 4 for s in result.steps)
checked = sliced("def f(x):\n if is_safe_url(x):\n redirect(x)", family="untrusted_destination")
assert not any(s.role == "guard" for s in checked.steps) # flat IR cannot prove dominance
def test_learn_slice_parameterized_sink_and_family_routing() -> None:
result = sliced('def f(x):\n execute("select ?", [x])')
assert {s.role for s in result.steps} == {"source", "sanitizer", "sink"}
assert sliced("def f(x):\n execute(x)", family="path").steps == ()
@pytest.mark.parametrize(
"language,code",
[
("javascript", "function f(req) { const x = req.query.x; const y = x; eval(y); }"),
("typescript", "function f(req: any) { const x = req.query.x; eval(x); }"),
("c", "void f(char *x) { char *y = x; system(y); }"),
("java", "class C { void f(String x) { String y = x; exec(y); } }"),
],
)
def test_learn_slice_shared_cst_declarations(language: str, code: str) -> None:
assert any(s.role == "sink" for s in sliced(code, language).steps)
def test_learn_slice_same_line_assignment_order() -> None:
assert not sliced('function f(x) { x = "safe"; eval(x); }', "javascript").steps
assert sliced('function f(x) { eval(x); x = "safe"; }', "javascript").steps
assert sliced("function f() { const x = req.query.x; eval(x); }", "javascript").steps
def test_learn_slice_fixed_framing_redaction_and_unsupported() -> None:
assert function_slice("<?php function f($x) { exec($x); }", "php", "injection") is None
assert render_slice(None, "php") == "Data flow inside the function: not analysed (php is not supported)."
assert render_slice(Slice(), "anything") == "Data flow inside the function: no source reaches a sink of this family."
assert "CVE" not in render_slice(None, "CVE-2026-1234")
result = sliced("def f(x):\n execute(x) # CVE-2026-1234 vulnerable")
shown = render_slice(result, "python")
assert shown.startswith("Data flow inside the function:\nL1 source:")
assert "CVE-2026-1234" not in shown and "vulnerable" not in shown
assert function_slice("def f(x):\n execute(", "python", "injection") is None
def test_learn_slice_default_prompt_frozen_before_change() -> None:
import test_learn_program as baseline
prompt, _ = baseline.program().prepare(baseline.candidate(), baseline.FOLD)
assert hashlib.sha256(json.dumps(prompt.messages, sort_keys=True).encode()).hexdigest() == (
"f6b5cfb5a8b7cb1591d7b4534e254ad2a349452a03e825bdcf250e5f60b4e80b"
)
assert baseline.ProgramSpec().slice is False
reserved, _ = baseline.program(slice=True).prepare(baseline.candidate(), baseline.FOLD)
assert reserved.messages == prompt.messages # no prompt wiring before audit review
@pytest.fixture
def audit_module() -> ModuleType:
path = Path(__file__).resolve().parents[1] / "benchmarks/learn/slice_leak_audit.py"
spec = importlib.util.spec_from_file_location("slice_leak_audit", path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_learn_slice_audit_metrics_ties_and_reverse_leak(audit_module: ModuleType) -> None:
rows = [
{"family": "injection", "group": "g", "fold": "f", "pair": str(pair), "label": label, "has_flow": 1 - label, "slice_length": 3}
for pair in range(10)
for label in (0, 1)
]
metric = audit_module.metrics(rows, "has_flow")
assert metric["within_pair_accuracy"] == metric["pooled_auc"] == 0
assert metric["flagged"] and metric["best_direction_pooled_auc"] == 1
metric = audit_module.metrics(rows, "slice_length")
assert metric["within_pair_accuracy"] == metric["pooled_auc"] == 0.5
assert not metric["flagged"]
assert audit_module.metrics([], "has_flow")["pooled_auc"] is None
def test_learn_slice_audit_reads_and_never_writes(audit_module: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None:
from learn_fixtures import corpus
raw = b" 10 def f(x):\n 11 execute(x)\n"
digest = hashlib.sha256(raw).hexdigest()
example = replace(corpus(groups=1)[0], excerpt_sha=digest, source="pairs", language="python", family="injection")
monkeypatch.setattr(audit_module, "load_examples", lambda store: [example])
class ReadOnly:
def get_blob(self, prefix: str, name: str, *, verify: bool = True) -> bytes:
assert (prefix, name, verify) == ("excerpts", digest, False)
return raw
units = [{"family": example.family, "label": example.label, "group": example.group, "unit": example.id, "pair": None, "fold": "f"}]
report = audit_module.audit(ReadOnly(), units, expected_rows=1, expected_pairs=0)
assert report["example_rows_read"] == report["matched_rows"] == report["excerpts_read"] == 1
assert report["excerpt_bytes"] == len(raw)
assert report["coverage"]["injection"]["flow"] == 1
class Corrupt(ReadOnly):
def get_blob(self, prefix: str, name: str, *, verify: bool = True) -> bytes:
assert not verify
return b"corrupt"
with pytest.raises(ValueError, match="corrupt excerpt"):
audit_module.audit(Corrupt(), units, expected_rows=1, expected_pairs=0)
def test_learn_slice_augmented_assignment_preserves_value() -> None:
result = sliced('def f(x):\n x += " suffix"\n execute(x)')
assert {s.line for s in result.steps} == {1, 2, 3}
result = sliced('function f(x) { x += " suffix"; eval(x); }', "javascript")
assert any(s.role == "sink" for s in result.steps)
def test_learn_slice_python_fallback_order(monkeypatch: pytest.MonkeyPatch) -> None:
from openultrasast.semantic import cst
monkeypatch.setattr(cst, "parse_with_cst", lambda *args: None)
assert sliced("def f():\n x = input(); execute(x)").steps
assert not sliced('def f(x):\n x = "safe"; execute(x)').steps
def test_learn_slice_bare_arrow_parameter() -> None:
assert sliced("const f = x => eval(x);", "javascript").steps