marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pair-corpus-honesty): function-scoped honest scorer with mechanisms and loss counters

Browse Source 
Tasks 1.2, 2.1-2.5 and 6.2 (pair-side filters).

- pairs.py: coverage with a source counts as detection and as leak;
  function-scoped matching by line-range containment from the vulnerable
  side's parse (unresolvable labels are counted loss, never silent
  misses); exact or alias sink matching; weak CWE-only labels never match;
  side-by-side inventory/overlay/hunter scorers; per_profile,
  per_mechanism, achievable, known_limit and loss payload; loader rejects
  unknown mechanism/provenance/split and reserved or missing function
  labels on harvested slices; candidate rulesets thread through for the
  improve loop.
- overlay.py: OverlayRecord.function (additive) filled from the shared
  naming helper in one parse; benchmark.py: ExpectedFinding.mechanism.
- cli pairs: --slice over every slice, --profile, --split, --hunter,
  --hunter-model.
- catalogs migrated to the new schema (provenance, split, function,
  mechanism, known_limit for owasp-java-hash); baseline recorded in the
  spec notes.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-05 10:45:43 +02:00
co-authored by Claude Fable 5.1
parent 8ab4207618
commit c0b9e41000
9 changed files with 1915 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+7
View File
@@ -3,3 +3,10 @@ OPENROUTER_API_KEY=
# Prove-budget and `ousast index` embeddings via OpenRouter /embeddings.
OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small
# Any OpenAI-compatible chat endpoint works for the hunter: set the key above to that
# provider's key and override the base URL. Example for DeepSeek:
# OPENROUTER_API_KEY=sk-deepseek-...
# OPENROUTER_BASE_URL=https://api.deepseek.com/v1
# Then set [models].hunter in openultrasast.toml (or --hunter-model) to that provider's model id.
OPENROUTER_BASE_URL=
benchmarks/pairs/catalog.toml
+18
View File
@@ -55,6 +55,8 @@ fixed = "github/sven-cwe-078-os-system/fixed.py"
relpath = "vuedj/configtitania/views.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -63,6 +65,7 @@ path = "vuedj/configtitania/views.py"
rule_id = "python-os-command"
sink = "os.system"
evidence = "os.system concatenates attacker-controlled username into a shell command"
mechanism = "source_reaches_sink"
[[pair]]
name = "sven-cwe-078-shell-true"
@@ -78,6 +81,8 @@ fixed = "github/sven-cwe-078-shell-true/fixed.py"
relpath = "mobly/controllers/android_device_lib/adb.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
[[pair.expected]]
cwe = "CWE-78"
@@ -86,6 +91,7 @@ path = "mobly/controllers/android_device_lib/adb.py"
rule_id = "python-shell-true"
sink = "subprocess"
evidence = "subprocess.Popen runs the command with shell=True"
mechanism = "source_reaches_sink"
[[pair]]
name = "sven-cwe-089-execute-percent"
@@ -101,6 +107,8 @@ fixed = "github/sven-cwe-089-execute-percent/fixed.py"
relpath = "cgi/common.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
[[pair.expected]]
cwe = "CWE-89"
@@ -109,6 +117,7 @@ path = "cgi/common.py"
rule_id = "python-sql-injection"
sink = "execute"
evidence = "execute percent-formats attacker-controlled input into SQL"
mechanism = "source_reaches_sink"
[[pair]]
name = "sven-cwe-089-query-concat"
@@ -124,6 +133,8 @@ fixed = "github/sven-cwe-089-query-concat/fixed.py"
relpath = "modules/query_lastfm.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
[[pair.expected]]
cwe = "CWE-89"
@@ -131,6 +142,7 @@ class = "sql injection"
path = "modules/query_lastfm.py"
sink = "SELECT"
evidence = "SQL query concatenates attacker-controlled tag_num"
mechanism = "source_reaches_sink"
[[pair]]
name = "hutool-cve-2018-17297"
@@ -149,6 +161,8 @@ fixed = "github/hutool-cve-2018-17297/fixed/ZipUtil.java"
relpath = "cn/hutool/core/util/ZipUtil.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
[[pair.expected]]
cwe = "CWE-22"
@@ -156,6 +170,7 @@ class = "path traversal"
path = "cn/hutool/core/util/ZipUtil.java"
sink = "ZipEntry"
evidence = "zip entry name is joined onto the extract directory without a zip-slip check"
mechanism = "path_join_user_input"
[[pair]]
name = "node-serialize-cve-2017-5941"
@@ -171,6 +186,8 @@ fixed = "github/node-serialize-cve-2017-5941/fixed.js"
relpath = "index.js"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
[[pair.expected]]
cwe = "CWE-95"
@@ -179,3 +196,4 @@ path = "index.js"
rule_id = "js-eval"
sink = "eval"
evidence = "eval executes attacker-controlled serialized input"
mechanism = "source_reaches_sink"
benchmarks/pairs/sast/catalog.toml
+34
View File
@@ -17,6 +17,8 @@ fixed = "owasp-python/BenchmarkTest00073.py"
relpath = "testcode/BenchmarkTest00074.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-94"
@@ -25,6 +27,7 @@ path = "testcode/BenchmarkTest00074.py"
rule_id = "python-unsafe-eval"
sink = "exec"
evidence = "exec executes attacker-controlled input"
mechanism = "container_taint"
[[pair]]
name = "owasp-python-deser"
@@ -39,6 +42,8 @@ fixed = "owasp-python/BenchmarkTest00079.py"
relpath = "testcode/BenchmarkTest00078.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-502"
@@ -47,6 +52,7 @@ path = "testcode/BenchmarkTest00078.py"
rule_id = "python-unsafe-deserialization"
sink = "pickle"
evidence = "pickle.loads deserializes untrusted input"
mechanism = "source_reaches_sink"
[[pair]]
name = "owasp-python-cmdi"
@@ -61,6 +67,8 @@ fixed = "owasp-python/BenchmarkTest00350.py"
relpath = "testcode/BenchmarkTest00165.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -69,6 +77,7 @@ path = "testcode/BenchmarkTest00165.py"
rule_id = "python-shell-true"
sink = "subprocess"
evidence = "subprocess runs an attacker-controlled command"
mechanism = "container_taint"
[[pair]]
name = "owasp-python-sqli"
@@ -83,6 +92,8 @@ fixed = "owasp-python/BenchmarkTest00011.py"
relpath = "testcode/BenchmarkTest00099.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-89"
@@ -91,6 +102,7 @@ path = "testcode/BenchmarkTest00099.py"
rule_id = "python-sql-injection"
sink = "execute"
evidence = "execute runs a SQL string built from request input"
mechanism = "container_taint"
[[pair]]
name = "owasp-python-hash"
@@ -105,6 +117,8 @@ fixed = "owasp-python/BenchmarkTest00055.py"
relpath = "testcode/BenchmarkTest00054.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-327"
@@ -113,6 +127,7 @@ path = "testcode/BenchmarkTest00054.py"
rule_id = "python-weak-hash"
sink = "md5"
evidence = "md5 is a weak hash"
mechanism = "config_sink_weak_literal"
# --- OWASP Benchmark Java v1.2 ---
@@ -129,6 +144,8 @@ fixed = "owasp-java/BenchmarkTest00090.java"
relpath = "org/owasp/benchmark/testcode/BenchmarkTest00007.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -137,6 +154,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00007.java"
rule_id = "java-command-exec"
sink = "exec"
evidence = "Runtime.exec runs an attacker-controlled command"
mechanism = "source_reaches_sink"
[[pair]]
name = "owasp-java-sqli"
@@ -151,6 +169,8 @@ fixed = "owasp-java/BenchmarkTest00052.java"
relpath = "org/owasp/benchmark/testcode/BenchmarkTest00008.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-89"
@@ -159,6 +179,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00008.java"
rule_id = "java-sql-injection"
sink = "prepareCall"
evidence = "SQL is concatenated with request input"
mechanism = "source_reaches_sink"
[[pair]]
name = "owasp-java-hash"
@@ -173,6 +194,9 @@ fixed = "owasp-java/BenchmarkTest00009.java"
relpath = "org/owasp/benchmark/testcode/BenchmarkTest00003.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
known_limit = "cross_artifact: OWASP labels this true via hashAlg1=MD5 in benchmark.properties, which is not in the vendored file"
[[pair.expected]]
cwe = "CWE-327"
@@ -181,6 +205,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00003.java"
rule_id = "java-weak-hash"
sink = "MessageDigest"
evidence = "MessageDigest uses a weak hash algorithm"
mechanism = "cross_artifact"
# --- Juliet C/C++ 1.3 (CC0) ---
@@ -197,6 +222,8 @@ fixed = "juliet-c/cwe134-printf-fixed.c"
relpath = "CWE134_char_console_printf_01.c"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-134"
@@ -205,6 +232,7 @@ path = "CWE134_char_console_printf_01.c"
rule_id = "c-unsafe-format"
sink = "printf"
evidence = "printf uses attacker-controlled format string"
mechanism = "source_reaches_sink"
[[pair]]
name = "juliet-c-cwe121-strcpy"
@@ -219,6 +247,8 @@ fixed = "juliet-c/cwe121-strcpy-fixed.c"
relpath = "CWE121_char_declare_cpy_01.c"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-120"
@@ -227,6 +257,7 @@ path = "CWE121_char_declare_cpy_01.c"
rule_id = "c-unsafe-copy"
sink = "strcpy"
evidence = "strcpy copies into a buffer without a null terminator"
mechanism = "fixed_buffer_capacity"
# --- Juliet Java 1.3 (CC0) ---
@@ -243,6 +274,8 @@ fixed = "juliet-java/cwe78-exec-fixed.java"
relpath = "CWE78_OS_Command_Injection__console_readLine_01.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -251,3 +284,4 @@ path = "CWE78_OS_Command_Injection__console_readLine_01.java"
rule_id = "java-command-exec"
sink = "exec"
evidence = "Runtime.exec concatenates attacker-controlled input into a command"
mechanism = "source_reaches_sink"
benchmarks/pairs/vfc/catalog.toml
+706 -2
View File
File diff suppressed because it is too large Load Diff
src/openultrasast/benchmark.py
+2
View File
@@ -26,6 +26,7 @@ class ExpectedFinding:
line: int | None = None
function: str | None = None
sink: str | None = None
mechanism: str | None = None
@dataclass(frozen=True)
@@ -161,6 +162,7 @@ def load_benchmark_manifest(path: Path) -> BenchmarkManifest:
line=int(item["line"]) if "line" in item else None,
function=str(item["function"]) if "function" in item else None,
sink=str(item["sink"]) if "sink" in item else None,
mechanism=str(item["mechanism"]) if "mechanism" in item else None,
)
for item in expected
],
src/openultrasast/cli.py
+47 -6
View File
@@ -41,7 +41,17 @@ from .improve import RoundOutcome, run_improvement
from .index import build_code_chunks
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
from .pair_gate import print_pair_metrics
from .pairs import DEFAULT_CATALOG, evaluate_catalog, load_pair_catalog, result_payload, select_slice
from .pairs import (
DEFAULT_CATALOG,
SLICE_NAMES,
evaluate_catalog,
load_pair_catalog,
make_hunter_scan,
result_payload,
select_profile,
select_slice,
select_split,
)
from .policy import assert_rules_resolve, load_policy
from .preprocess import preprocess_repository, write_preprocess_artifact
from .provider.openrouter import OpenRouterEmbeddingClient, OpenRouterError
@@ -125,7 +135,13 @@ def main(argv: list[str] | None = None) -> int:
help="scan isolated vuln-vs-fixed pairs (TP on vuln, silent on fix) and emit improve-loop signals",
)
pairs.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
pairs.add_argument("--slice", choices=("all", "local", "github", "sast", "vfc"), default="all")
pairs.add_argument("--slice", choices=SLICE_NAMES, default="all")
pairs.add_argument(
"--profile", choices=("all", "human", "agent", "mixed", "synthetic"), default="all", help="filter pairs by provenance profile"
)
pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split")
pairs.add_argument("--hunter", action="store_true", help="also score the LLM tool hunter on overlay slices (needs a hunter model)")
pairs.add_argument("--hunter-model", default=None, help="model id for --hunter (default: [models].hunter from openultrasast.toml)")
pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON")
subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
@@ -149,7 +165,15 @@ def main(argv: list[str] | None = None) -> int:
dry_run=args.dry_run,
)
if args.command == "pairs":
return _pairs(args.catalog, args.slice, json_out=args.json)
return _pairs(
args.catalog,
args.slice,
json_out=args.json,
profile=args.profile,
split=args.split,
hunter=args.hunter,
hunter_model=args.hunter_model,
)
if args.command == "mcp":
from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional
@@ -724,11 +748,28 @@ def _improve(
return 0
def _pairs(catalog: Path, slice_name: str, *, json_out: bool) -> int:
def _pairs(
catalog: Path,
slice_name: str,
*,
json_out: bool,
profile: str = "all",
split: str = "all",
hunter: bool = False,
hunter_model: str | None = None,
) -> int:
if not catalog.exists() or not catalog.is_file():
raise SystemExit(f"pair catalog is not a file: {catalog}")
cases = select_slice(load_pair_catalog(catalog), slice_name)
result = evaluate_catalog(cases)
cases = select_split(select_profile(select_slice(load_pair_catalog(catalog), slice_name), profile), split)
scan = None
if hunter:
config_path = Path("openultrasast.toml")
config = load_config(config_path if config_path.exists() else None)
model = hunter_model or config.models.hunter or ""
client = tool_hunter.resolve_hunter_client()
if model and client is not None:
scan = make_hunter_scan(client, model)
result = evaluate_catalog(cases, hunter=scan)
if json_out:
print(json.dumps(result_payload(result), indent=2, sort_keys=True))
return 0
src/openultrasast/pairs.py
+517 -110
View File
@@ -3,6 +3,12 @@
Isolated trees keep the two sides from contaminating each other (a `safe.py`
sitting next to `app.py` cannot measure false positives). Pair outcomes feed
the improvement loop as miss (vuln) and fp (fix) signals.
pair-corpus-honesty: the overlay scorer counts ``coverage`` records with a
source as detections and as leaks, matches by function when a label names one,
flags CWE-only rows as weak labels, reports parser/adjudication loss, and
slices metrics by provenance profile and by mechanism. Known-limit pairs are
evaluated but excluded from the achievable numbers.
"""
from __future__ import annotations
@@ -10,8 +16,9 @@ from __future__ import annotations
import shutil
import tempfile
import tomllib
from collections.abc import Sequence
from dataclasses import asdict, dataclass, field
from collections.abc import Callable, Mapping, Sequence
from dataclasses import asdict, dataclass, field, replace
from functools import lru_cache
from pathlib import Path
from .benchmark import (
@@ -24,15 +31,38 @@ from .benchmark import (
)
from .findings import StaticFinding, quick_scan_findings
from .mapping import analyze_entry_points, attach_reachability_hints
from .preprocess import preprocess_repository
from .preprocess import FileTarget, preprocess_repository
from .rank import rank_targets
from .ruleset import PatternRule
from .semantic import OverlayRecord, adjudicate
from .semantic.facts import FactLoadError, load_facts
from .semantic.functions import RESERVED_FUNCTION_LABELS, function_at, named_function_ranges, spans_named
DEFAULT_CATALOG = Path("benchmarks/pairs/catalog.toml")
DEFAULT_SAST_CATALOG = Path("benchmarks/pairs/sast/catalog.toml")
DEFAULT_VFC_CATALOG = Path("benchmarks/pairs/vfc/catalog.toml")
DEFAULT_VIBE_PY_CATALOG = Path("benchmarks/pairs/vibe-py/catalog.toml")
DEFAULT_VFC_JS_CATALOG = Path("benchmarks/pairs/vfc-js/catalog.toml")
DEFAULT_AGENT_VFC_CATALOG = Path("benchmarks/pairs/agent-vfc/catalog.toml")
DEFAULT_DATASETS = Path("benchmarks/pairs/datasets.toml")
OVERLAY_SLICES = frozenset({"sast", "vfc"})
DEFAULT_MECHANISMS = Path("benchmarks/pairs/mechanisms.toml")
# Slices scored on overlay dispositions (inventory fallback when nothing adjudicates).
OVERLAY_SLICES = frozenset({"sast", "vfc", "vibe-py", "vfc-js", "agent-vfc"})
# Harvested function slices: every label must name the function (design §LabelSchema); "<anon>" does not scope.
FUNCTION_REQUIRED_SLICES = OVERLAY_SLICES - {"sast"}
FunctionRanges = dict[str, tuple[tuple[str, int, int], ...]]
SLICE_NAMES = ("all", "local", "github", "sast", "vfc", "vibe-py", "vfc-js", "agent-vfc")
PROVENANCES = frozenset({"human", "agent", "mixed", "synthetic"})
SPLITS = frozenset({"train", "holdout"})
_DEFAULT_PROVENANCE = {"sast": "synthetic", "local": "human", "github": "human", "vfc": "human"}
_MANIFEST_MECHANISM = "source_reaches_sink" # cheat-sheet fixtures referenced via expected_from
HunterScan = Callable[[Path], list[StaticFinding]]
class CatalogError(ValueError):
"""Raised when a pair catalog violates the label schema."""
@dataclass(frozen=True)
@@ -52,6 +82,9 @@ class PairCase:
commit_url: str = ""
cve: str = ""
license: str = ""
provenance: str = "human"
split: str = "train"
known_limit: str | None = None
@dataclass(frozen=True)
@@ -73,6 +106,17 @@ class PairOutcome:
leaks: tuple[str, ...]
commit_url: str = ""
cve: str = ""
provenance: str = "human"
mechanisms: tuple[str, ...] = ()
known_limit: str | None = None
split: str = "train"
weak_labels: int = 0
parse_failed_vuln: int = 0
parse_failed_fixed: int = 0
unadjudicated_vuln: int = 0
unadjudicated_fixed: int = 0
detection_kinds: tuple[str, ...] = ()
unresolved_labels: int = 0 # expected rows whose function names no range on the vulnerable side
@dataclass(frozen=True)
@@ -98,6 +142,12 @@ class PairEvalResult:
per_slice: dict[str, PairCorpusMetrics]
signals: tuple[dict[str, object], ...]
scorers: dict[str, dict[str, PairCorpusMetrics]] = field(default_factory=dict)
per_profile: dict[str, PairCorpusMetrics] = field(default_factory=dict)
per_mechanism: dict[str, PairCorpusMetrics] = field(default_factory=dict)
achievable: dict[str, PairCorpusMetrics] = field(default_factory=dict)
known_limit: tuple[str, ...] = ()
loss: dict[str, dict[str, int]] = field(default_factory=dict)
degradations: tuple[dict[str, object], ...] = ()
def load_datasets(path: Path = DEFAULT_DATASETS) -> tuple[dict[str, object], ...]:
@@ -105,13 +155,30 @@ def load_datasets(path: Path = DEFAULT_DATASETS) -> tuple[dict[str, object], ...
return tuple(dict(item) for item in payload.get("dataset", []))
@lru_cache(maxsize=4)
def load_mechanisms(path: Path = DEFAULT_MECHANISMS) -> tuple[str, frozenset[str]]:
"""Closed mechanism vocabulary: (version, ids). Missing file is a catalog error."""
if not path.is_file():
raise CatalogError(f"mechanism vocabulary missing: {path}")
payload = tomllib.loads(path.read_text())
ids = frozenset(str(item["id"]) for item in payload.get("mechanism", []) if isinstance(item, dict) and item.get("id"))
if not ids:
raise CatalogError(f"mechanism vocabulary is empty: {path}")
return str(payload.get("version", "1")), ids
def load_pair_catalog(path: Path = DEFAULT_CATALOG) -> tuple[PairCase, ...]:
cases = _load_catalog_file(path)
if path.resolve() == DEFAULT_CATALOG.resolve():
if DEFAULT_SAST_CATALOG.exists():
cases = cases + _load_catalog_file(DEFAULT_SAST_CATALOG)
if DEFAULT_VFC_CATALOG.exists():
cases = cases + _load_catalog_file(DEFAULT_VFC_CATALOG)
for extra in (
DEFAULT_SAST_CATALOG,
DEFAULT_VFC_CATALOG,
DEFAULT_VIBE_PY_CATALOG,
DEFAULT_VFC_JS_CATALOG,
DEFAULT_AGENT_VFC_CATALOG,
):
if extra.exists():
cases = cases + _load_catalog_file(extra)
return cases
@@ -119,13 +186,29 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]:
catalog_path = path.resolve()
payload = tomllib.loads(catalog_path.read_text())
root = catalog_path.parent
_, mechanisms = load_mechanisms()
cases: list[PairCase] = []
for item in payload.get("pair", []):
expected = _expected_for(item, root)
name = str(item["name"])
slice_name = str(item.get("slice", "github"))
expected = _expected_for(item, root, name, mechanisms)
if slice_name in FUNCTION_REQUIRED_SLICES:
for row in expected:
if not row.function or row.function in RESERVED_FUNCTION_LABELS:
raise CatalogError(
f"pair {name}: slice {slice_name} requires a named function on every expected row (got {row.function!r})"
)
provenance = str(item.get("provenance") or _DEFAULT_PROVENANCE.get(slice_name, "human"))
if provenance not in PROVENANCES:
raise CatalogError(f"pair {name}: unknown provenance {provenance!r}; expected one of {sorted(PROVENANCES)}")
split = str(item.get("split", "train"))
if split not in SPLITS:
raise CatalogError(f"pair {name}: unknown split {split!r}; expected train or holdout")
known_limit = item.get("known_limit")
cases.append(
PairCase(
name=str(item["name"]),
slice=str(item.get("slice", "github")),
name=name,
slice=slice_name,
language=str(item.get("language", "")),
origin=str(item.get("origin", "")),
vuln_file=_resolve(root, str(item["vuln"])),
@@ -139,30 +222,50 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]:
commit_url=str(item.get("commit_url", "")),
cve=str(item.get("cve", "")),
license=str(item.get("license", "")),
provenance=provenance,
split=split,
known_limit=str(known_limit) if known_limit else None,
)
)
return tuple(cases)
def evaluate_pair(case: PairCase) -> PairOutcome:
def evaluate_pair(case: PairCase, *, hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome:
with tempfile.TemporaryDirectory(prefix="ousast-pair-") as scratch:
vuln_root = _materialize(Path(scratch) / "vuln", case.vuln_file, case.relpath)
fix_root = _materialize(Path(scratch) / "fixed", case.fixed_file, case.relpath)
if hunter is not None:
return _evaluate_hunter_pair(case, vuln_root, fix_root, hunter)
if case.slice in OVERLAY_SLICES:
return _evaluate_overlay_pair(case, vuln_root, fix_root)
return _evaluate_inventory_pair(case, vuln_root, fix_root)
return _evaluate_overlay_pair(case, vuln_root, fix_root, ruleset)
return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset)
def evaluate_catalog(cases: Sequence[PairCase]) -> PairEvalResult:
outcomes = tuple(evaluate_pair(case) for case in cases)
def evaluate_catalog(
cases: Sequence[PairCase],
*,
hunter: HunterScan | None = None,
ruleset: tuple[PatternRule, ...] | None = None,
inventory_sidecar: bool = True,
) -> PairEvalResult:
"""Score every case. ``ruleset`` lets the improve loop score a candidate ledger; ``inventory_sidecar=False`` skips the second scan."""
outcomes = tuple(evaluate_pair(case, ruleset=ruleset) for case in cases)
scorers: dict[str, dict[str, PairCorpusMetrics]] = {}
vfc_cases = tuple(case for case in cases if case.slice == "vfc")
if vfc_cases:
overlay_outcomes = tuple(item for item in outcomes if item.slice == "vfc")
scorers["vfc"] = {
"overlay": _metrics(overlay_outcomes),
"inventory": _metrics(tuple(_inventory_only(case) for case in vfc_cases)),
}
degradations: list[dict[str, object]] = []
for slice_name in sorted({case.slice for case in cases if case.slice in OVERLAY_SLICES}):
slice_cases = tuple(case for case in cases if case.slice == slice_name)
scorers[slice_name] = {"overlay": _metrics(tuple(item for item in outcomes if item.slice == slice_name))}
if inventory_sidecar:
scorers[slice_name]["inventory"] = _metrics(tuple(_inventory_only(case, ruleset) for case in slice_cases))
if hunter is not None:
scorers[slice_name]["hunter"] = _metrics(tuple(evaluate_pair(case, hunter=hunter) for case in slice_cases))
if hunter is None and scorers:
degradations.append({"stage": "pairs", "reason": "hunter_model_unavailable"})
achievable_outcomes = tuple(item for item in outcomes if item.known_limit is None)
per_mechanism: dict[str, list[PairOutcome]] = {}
for item in outcomes:
for mechanism in item.mechanisms:
per_mechanism.setdefault(mechanism, []).append(item)
return PairEvalResult(
outcomes=outcomes,
overall=_metrics(outcomes),
@@ -172,19 +275,39 @@ def evaluate_catalog(cases: Sequence[PairCase]) -> PairEvalResult:
},
signals=tuple(build_pair_signals(outcomes)),
scorers=scorers,
per_profile={
profile: _metrics(tuple(item for item in outcomes if item.provenance == profile))
for profile in sorted({item.provenance for item in outcomes})
},
per_mechanism={mechanism: _metrics(tuple(items)) for mechanism, items in sorted(per_mechanism.items())},
achievable={
slice_name: _metrics(tuple(item for item in achievable_outcomes if item.slice == slice_name))
for slice_name in sorted({item.slice for item in achievable_outcomes})
},
known_limit=tuple(item.name for item in outcomes if item.known_limit is not None),
loss=_loss(outcomes),
degradations=tuple(degradations),
)
def build_pair_signals(outcomes: Sequence[PairOutcome]) -> list[dict[str, object]]:
"""Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name."""
"""Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms."""
signals: list[dict[str, object]] = []
for outcome in outcomes:
if outcome.known_limit is not None:
continue # unachievable by construction; the loop must not chase it
common: dict[str, object] = {
"pair": outcome.name,
"profile": outcome.provenance,
"mechanisms": list(outcome.mechanisms),
"split": outcome.split,
}
for miss in outcome.misses:
rule_id, path, cwe = (miss.split(":", 2) + ["", ""])[:3]
signals.append({"rule_id": rule_id if rule_id != "-" else "", "signal": "miss", "cwe": cwe, "path": path, "pair": outcome.name})
signals.append({"rule_id": rule_id if rule_id != "-" else "", "signal": "miss", "cwe": cwe, "path": path, **common})
for leak in outcome.leaks:
rule_id = leak.split(":", 1)[0]
signals.append({"rule_id": rule_id, "signal": "fp", "path": leak, "pair": outcome.name, "side": "fixed"})
signals.append({"rule_id": rule_id, "signal": "fp", "path": leak, "side": "fixed", **common})
return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("pair"))))
@@ -194,6 +317,18 @@ def select_slice(cases: Sequence[PairCase], slice_name: str | None) -> tuple[Pai
return tuple(case for case in cases if case.slice == slice_name)
def select_profile(cases: Sequence[PairCase], profile: str | None) -> tuple[PairCase, ...]:
if not profile or profile == "all":
return tuple(cases)
return tuple(case for case in cases if case.provenance == profile)
def select_split(cases: Sequence[PairCase], split: str | None) -> tuple[PairCase, ...]:
if not split or split == "all":
return tuple(cases)
return tuple(case for case in cases if case.split == split)
def result_payload(result: PairEvalResult) -> dict[str, object]:
return {
"overall": asdict(result.overall),
@@ -201,16 +336,27 @@ def result_payload(result: PairEvalResult) -> dict[str, object]:
"outcomes": [asdict(outcome) for outcome in result.outcomes],
"signals": list(result.signals),
"scorers": {
slice_name: {scorer: asdict(metrics) for scorer, metrics in inner.items()}
for slice_name, inner in result.scorers.items()
slice_name: {scorer: asdict(metrics) for scorer, metrics in inner.items()} for slice_name, inner in result.scorers.items()
},
"per_profile": {name: asdict(metrics) for name, metrics in result.per_profile.items()},
"per_mechanism": {name: asdict(metrics) for name, metrics in result.per_mechanism.items()},
"achievable": {name: asdict(metrics) for name, metrics in result.achievable.items()},
"known_limit": list(result.known_limit),
"loss": {name: dict(counts) for name, counts in result.loss.items()},
"degradations": list(result.degradations),
}
def _expected_for(item: dict[str, object], root: Path) -> tuple[ExpectedFinding, ...]:
# --- catalog helpers -------------------------------------------------------
def _expected_for(item: dict[str, object], root: Path, name: str, mechanisms: frozenset[str]) -> tuple[ExpectedFinding, ...]:
inline = item.get("expected")
if isinstance(inline, list) and inline:
return tuple(_parse_expected(entry) for entry in inline if isinstance(entry, dict))
rows = tuple(_parse_expected(entry, name, mechanisms) for entry in inline if isinstance(entry, dict))
for row in rows:
_validate_expected(row, name)
return rows
expected_from = item.get("expected_from")
if not expected_from:
return ()
@@ -218,11 +364,24 @@ def _expected_for(item: dict[str, object], root: Path) -> tuple[ExpectedFinding,
relpath = str(item["relpath"])
# Pair eval measures ruled sinks. Unruled planted misses (split-sink, CWE-190)
# stay on the stage-1/2 fixtures and must not drag local pair_correct below 1.0.
return tuple(entry for entry in manifest.expected if entry.rule_id and (relpath in entry.path or entry.path in relpath))
derived: list[ExpectedFinding] = []
for entry in manifest.expected:
if not (entry.rule_id and (relpath in entry.path or entry.path in relpath)):
continue
mechanism = entry.mechanism or _MANIFEST_MECHANISM
if mechanism not in mechanisms:
raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r} in {expected_from}")
derived.append(ExpectedFinding(**{**asdict(entry), "mechanism": mechanism}))
return tuple(derived)
def _parse_expected(item: dict[str, object]) -> ExpectedFinding:
def _parse_expected(item: dict[str, object], name: str, mechanisms: frozenset[str]) -> ExpectedFinding:
line_value = item.get("line")
mechanism = item.get("mechanism")
if not isinstance(mechanism, str) or not mechanism:
raise CatalogError(f"pair {name}: expected row is missing mechanism")
if mechanism not in mechanisms:
raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r}; see benchmarks/pairs/mechanisms.toml")
return ExpectedFinding(
cwe=str(item["cwe"]),
vulnerability_class=str(item.get("class", item.get("vulnerability_class", "unknown"))),
@@ -232,10 +391,21 @@ def _parse_expected(item: dict[str, object]) -> ExpectedFinding:
line=line_value if isinstance(line_value, int) else None,
function=str(item["function"]) if "function" in item else None,
sink=str(item["sink"]) if "sink" in item else None,
mechanism=mechanism,
)
def _manifest(case: PairCase) -> BenchmarkManifest:
def _validate_expected(row: ExpectedFinding, name: str) -> None:
if (row.sink or "").lower() == "unknown" and not row.function and not row.rule_id:
raise CatalogError(f"pair {name}: expected row has sink=unknown and neither function nor rule_id; label it or drop it")
def is_weak_label(row: ExpectedFinding) -> bool:
"""A row that can only match by CWE string. Never counts as a detection."""
return not row.function and not row.rule_id and (not row.sink or row.sink.lower() == "unknown")
def _manifest(case: PairCase, expected: Sequence[ExpectedFinding] | None = None) -> BenchmarkManifest:
return BenchmarkManifest(
name=case.name,
language=case.language,
@@ -243,7 +413,7 @@ def _manifest(case: PairCase) -> BenchmarkManifest:
setup=[],
source=BenchmarkSource(type="local", path=str(case.vuln_file)),
modes=["quick"],
expected=list(case.expected),
expected=list(case.expected if expected is None else expected),
known_noise=[],
baselines=[],
)
@@ -261,31 +431,127 @@ def _materialize(dest: Path, source: Path, relpath: str) -> Path:
return dest
def _quick_scan(target: Path) -> list[StaticFinding]:
_, targets = preprocess_repository(target)
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
return quick_scan_findings(target, targets, rank_targets(targets))
# --- scans -----------------------------------------------------------------
def _overlay_scan(target: Path) -> tuple[list[StaticFinding], list[OverlayRecord]]:
_, targets = preprocess_repository(target)
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
findings = quick_scan_findings(target, targets, rank_targets(targets))
return findings, adjudicate(root=target, targets=targets, findings=findings)
def _targets(root: Path) -> list[FileTarget]:
_, targets = preprocess_repository(root)
return attach_reachability_hints(targets, analyze_entry_points(root, targets))
def _inventory_only(case: PairCase) -> PairOutcome:
def _quick_scan(target: Path, ruleset: tuple[PatternRule, ...] | None = None) -> list[StaticFinding]:
targets = _targets(target)
findings = quick_scan_findings(target, targets, rank_targets(targets), ruleset)
return [finding for finding in findings if finding.status != "shadow"]
@dataclass(frozen=True)
class _Side:
findings: list[StaticFinding]
records: list[OverlayRecord]
parse_failed: int
ranges: dict[str, tuple[tuple[str, int, int], ...]]
def _overlay_scan(target: Path, ruleset: tuple[PatternRule, ...] | None = None) -> _Side:
targets = _targets(target)
findings = [f for f in quick_scan_findings(target, targets, rank_targets(targets), ruleset) if f.status != "shadow"]
records = adjudicate(root=target, targets=targets, findings=findings)
parse_failed = 0
ranges: dict[str, tuple[tuple[str, int, int], ...]] = {}
for item in targets:
try:
text = (target / item.path).read_text(errors="ignore")
except OSError:
parse_failed += 1
continue
named = named_function_ranges(item.path, text, item.language)
if named is None:
parse_failed += 1
else:
ranges[item.path] = named
return _Side(findings=findings, records=records, parse_failed=parse_failed, ranges=ranges)
def _inventory_only(case: PairCase, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome:
with tempfile.TemporaryDirectory(prefix="ousast-pair-inv-") as scratch:
vuln_root = _materialize(Path(scratch) / "vuln", case.vuln_file, case.relpath)
fix_root = _materialize(Path(scratch) / "fixed", case.fixed_file, case.relpath)
return _evaluate_inventory_pair(case, vuln_root, fix_root)
return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset)
def _evaluate_inventory_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> PairOutcome:
vuln_findings = _quick_scan(vuln_root)
fix_findings = _quick_scan(fix_root)
def _evaluate_inventory_pair(
case: PairCase,
vuln_root: Path,
fix_root: Path,
*,
extra: Mapping[str, int] | None = None,
ruleset: tuple[PatternRule, ...] | None = None,
) -> PairOutcome:
vuln_findings = _with_functions(vuln_root, _quick_scan(vuln_root, ruleset))
fix_findings = _with_functions(fix_root, _quick_scan(fix_root, ruleset))
return _inventory_outcome(case, vuln_findings, fix_findings, extra=extra)
def _with_functions(root: Path, findings: list[StaticFinding]) -> list[StaticFinding]:
"""Give regex findings an enclosing function from a parse so function-labeled rows can match (Req 1.2)."""
if not any(finding.function_name is None for finding in findings):
return findings
ranges: dict[str, tuple[tuple[str, int, int], ...]] = {}
for target in _targets(root):
try:
named = named_function_ranges(target.path, (root / target.path).read_text(errors="ignore"), target.language)
except OSError:
continue
if named is not None:
ranges[target.path] = named
out: list[StaticFinding] = []
for finding in findings:
if finding.function_name is None:
function = function_at(ranges.get(finding.path, ()), finding.line)
if function is not None:
finding = replace(finding, function_name=function)
out.append(finding)
return out
def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None) -> HunterScan:
"""Run the tool hunter over every file of an isolated pair tree (Req 1.7)."""
from .complexity.map import Hotspot
from .tool_hunter import DEFAULT_MAX_STEPS, run_tool_hunter
def scan(root: Path) -> list[StaticFinding]:
hotspots = [
Hotspot(
path=target.path,
function_name=None,
score=1.0,
band="high",
signals={},
rationale="pair corpus function",
test_hint=None,
inventory_finding_ids=(),
)
for target in _targets(root)
]
return _with_functions(root, run_tool_hunter(root, hotspots, client=client, model=model, max_steps=max_steps or DEFAULT_MAX_STEPS)) # type: ignore[arg-type]
return scan
def _inventory_outcome(
case: PairCase,
vuln_findings: list[StaticFinding],
fix_findings: list[StaticFinding],
*,
extra: Mapping[str, int] | None = None,
) -> PairOutcome:
# Weak (CWE-only) rows never match, on this path either (Req 1.3): score the strong rows
# with the benchmark matcher and count the weak rows as misses.
strong = tuple(row for row in case.expected if not is_weak_label(row))
weak = tuple(row for row in case.expected if is_weak_label(row))
vuln_result = evaluate_benchmark(
run=BenchmarkRun(benchmark_run_id="pair", root=Path("/tmp/pair"), manifest=_manifest(case)),
run=BenchmarkRun(benchmark_run_id="pair", root=Path("/tmp/pair"), manifest=_manifest(case, strong)),
mode="quick",
findings=vuln_findings,
scan_id=None,
@@ -297,8 +563,11 @@ def _evaluate_inventory_pair(case: PairCase, vuln_root: Path, fix_root: Path) ->
leaks = _fix_leaks(case, fix_findings)
detected = recall + 1e-12 >= case.min_recall
silent = not leaks
misses = tuple(f"{miss.rule_id or '-'}:{miss.path}:{miss.cwe}" for miss in vuln_result.misses)
misses = tuple(f"{miss.rule_id or '-'}:{miss.path}:{miss.cwe}" for miss in vuln_result.misses) + tuple(
f"{row.rule_id or '-'}:{row.path}:{row.cwe}" for row in weak
)
leak_ids = tuple(finding.finding_id for finding in leaks)
counters = dict(extra or {})
return PairOutcome(
name=case.name,
slice=case.slice,
@@ -317,61 +586,47 @@ def _evaluate_inventory_pair(case: PairCase, vuln_root: Path, fix_root: Path) ->
leaks=leak_ids,
commit_url=case.commit_url,
cve=case.cve,
provenance=case.provenance,
mechanisms=_mechanisms(case),
known_limit=case.known_limit,
split=case.split,
weak_labels=len(weak),
parse_failed_vuln=int(counters.get("parse_failed_vuln", 0)),
parse_failed_fixed=int(counters.get("parse_failed_fixed", 0)),
unadjudicated_vuln=int(counters.get("unadjudicated_vuln", 0)),
unadjudicated_fixed=int(counters.get("unadjudicated_fixed", 0)),
unresolved_labels=int(counters.get("unresolved_labels", 0)),
detection_kinds=("inventory",) if vuln_findings else (),
)
def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> PairOutcome:
vuln_findings, vuln_overlay = _overlay_scan(vuln_root)
fix_findings, fix_overlay = _overlay_scan(fix_root)
if not _adjudicated(vuln_overlay) and not _adjudicated(fix_overlay):
def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome:
vuln = _overlay_scan(vuln_root, ruleset)
fixed = _overlay_scan(fix_root, ruleset)
loss: dict[str, int] = {
"parse_failed_vuln": vuln.parse_failed,
"parse_failed_fixed": fixed.parse_failed,
"unadjudicated_vuln": sum(1 for record in vuln.records if record.disposition == "unadjudicated"),
"unadjudicated_fixed": sum(1 for record in fixed.records if record.disposition == "unadjudicated"),
}
if not _adjudicated(vuln.records) and not _adjudicated(fixed.records):
# Parser/facts could not adjudicate this language; score inventory so
# labeled calibration does not collapse when tree-sitter grammars are absent.
vuln_result = evaluate_benchmark(
run=BenchmarkRun(benchmark_run_id="pair", root=Path("/tmp/pair"), manifest=_manifest(case)),
mode="quick",
findings=vuln_findings,
scan_id=None,
scan_run_dir=None,
)
expected_total = len(case.expected)
matched = vuln_result.metrics.matched_findings_total
recall = matched / expected_total if expected_total else 1.0
inventory_leaks = _fix_leaks(case, fix_findings)
detected = recall + 1e-12 >= case.min_recall
silent = not inventory_leaks
misses = tuple(f"{miss.rule_id or '-'}:{miss.path}:{miss.cwe}" for miss in vuln_result.misses)
leak_ids = tuple(finding.finding_id for finding in inventory_leaks)
return PairOutcome(
name=case.name,
slice=case.slice,
language=case.language,
origin=case.origin,
detected_vuln=detected,
silent_fix=silent,
pair_correct=detected and silent,
vuln_matched=matched,
vuln_expected=expected_total,
vuln_findings=len(vuln_findings),
fix_findings=len(fix_findings),
fix_leaks=len(inventory_leaks),
recall=recall,
misses=misses,
leaks=leak_ids,
commit_url=case.commit_url,
cve=case.cve,
)
promoted_vuln = [record for record in vuln_overlay if record.disposition == "promote"]
promoted_fix = [record for record in fix_overlay if record.disposition == "promote"]
loss["unresolved_labels"] = sum(1 for row in case.expected if _label_unresolved(row, vuln.ranges))
return _inventory_outcome(case, _with_functions(vuln_root, vuln.findings), _with_functions(fix_root, fixed.findings), extra=loss)
detected_vuln = _detections(vuln.records)
detected_fix = _detections(fixed.records)
expected_total = len(case.expected)
matched = _overlay_matches(case, promoted_vuln)
matched = sum(1 for row in case.expected if _overlay_matches_expected(row, detected_vuln, case.language, vuln.ranges))
recall = matched / expected_total if expected_total else 1.0
detected = recall + 1e-12 >= case.min_recall
overlay_leaks = _overlay_leaks(case, promoted_fix)
overlay_leaks = _overlay_leaks(case, detected_fix, fixed.ranges)
silent = not overlay_leaks
unresolved = sum(1 for row in case.expected if _label_unresolved(row, vuln.ranges))
misses = tuple(
f"{expected.rule_id or '-'}:{expected.path}:{expected.cwe}"
for expected in case.expected
if not _overlay_matches_expected(expected, promoted_vuln)
f"{row.rule_id or '-'}:{row.path}:{row.cwe}" + ("!unresolved" if _label_unresolved(row, vuln.ranges) else "")
for row in case.expected
if not _overlay_matches_expected(row, detected_vuln, case.language, vuln.ranges)
)
return PairOutcome(
name=case.name,
@@ -383,44 +638,174 @@ def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> P
pair_correct=detected and silent,
vuln_matched=matched,
vuln_expected=expected_total,
vuln_findings=len(promoted_vuln),
fix_findings=len(promoted_fix),
vuln_findings=len(detected_vuln),
fix_findings=len(detected_fix),
fix_leaks=len(overlay_leaks),
recall=recall,
misses=misses,
leaks=overlay_leaks,
commit_url=case.commit_url,
cve=case.cve,
provenance=case.provenance,
mechanisms=_mechanisms(case),
known_limit=case.known_limit,
split=case.split,
weak_labels=sum(1 for row in case.expected if is_weak_label(row)),
detection_kinds=tuple(sorted({record.disposition for record in detected_vuln})),
parse_failed_vuln=loss["parse_failed_vuln"],
parse_failed_fixed=loss["parse_failed_fixed"],
unadjudicated_vuln=loss["unadjudicated_vuln"],
unadjudicated_fixed=loss["unadjudicated_fixed"],
unresolved_labels=unresolved,
)
def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunter: HunterScan) -> PairOutcome:
"""Score the LLM hunter with the same detection and leak rules (Req 1.7)."""
vuln_findings = hunter(vuln_root)
fix_findings = hunter(fix_root)
vuln_ranges = _overlay_scan(vuln_root).ranges
fix_ranges = _overlay_scan(fix_root).ranges
expected_total = len(case.expected)
matched = sum(1 for row in case.expected if any(_hunter_matches_expected(row, finding, vuln_ranges) for finding in vuln_findings))
recall = matched / expected_total if expected_total else 1.0
detected = recall + 1e-12 >= case.min_recall
if case.fix_policy == "silent":
leaks = tuple(finding.finding_id for finding in fix_findings)
else:
leaks = tuple(
finding.finding_id
for finding in fix_findings
if any(_hunter_matches_expected(row, finding, fix_ranges) for row in case.expected)
)
misses = tuple(
f"{row.rule_id or '-'}:{row.path}:{row.cwe}"
for row in case.expected
if not any(_hunter_matches_expected(row, finding, vuln_ranges) for finding in vuln_findings)
)
return PairOutcome(
name=case.name,
slice=case.slice,
language=case.language,
origin=case.origin,
detected_vuln=detected,
silent_fix=not leaks,
pair_correct=detected and not leaks,
vuln_matched=matched,
vuln_expected=expected_total,
vuln_findings=len(vuln_findings),
fix_findings=len(fix_findings),
fix_leaks=len(leaks),
recall=recall,
misses=misses,
leaks=leaks,
commit_url=case.commit_url,
cve=case.cve,
provenance=case.provenance,
mechanisms=_mechanisms(case),
known_limit=case.known_limit,
split=case.split,
weak_labels=sum(1 for row in case.expected if is_weak_label(row)),
detection_kinds=("hunter",) if vuln_findings else (),
)
# --- matching ---------------------------------------------------------------
def _adjudicated(records: Sequence[OverlayRecord]) -> bool:
return any(record.disposition in {"promote", "demote", "coverage"} for record in records)
def _overlay_matches(case: PairCase, promoted: Sequence[OverlayRecord]) -> int:
return sum(1 for expected in case.expected if _overlay_matches_expected(expected, promoted))
def _detections(records: Sequence[OverlayRecord]) -> list[OverlayRecord]:
"""Promotions plus coverage records that carry a source-to-sink flow (Req 1.1)."""
return [
record
for record in records
if record.disposition == "promote" or (record.disposition == "coverage" and record.sources and record.sinks)
]
def _overlay_matches_expected(expected: ExpectedFinding, promoted: Sequence[OverlayRecord]) -> bool:
for record in promoted:
@lru_cache(maxsize=1)
def _sink_ids_by_rule() -> dict[tuple[str, str], frozenset[str]]:
"""(language, rule_id) -> fact sink ids that declare that rule (facts are the alias table)."""
try:
facts = load_facts()
except FactLoadError:
return {}
table: dict[tuple[str, str], set[str]] = {}
for sink in facts.sinks:
for rule_id in sink.rule_ids:
table.setdefault((sink.language, rule_id), set()).add(sink.id)
return {key: frozenset(value) for key, value in table.items()}
def _fact_language(language: str) -> str:
return {"c_cpp": "c", "cpp": "c", "typescript": "javascript"}.get(language, language)
def _in_named_function(record: OverlayRecord, function: str, ranges: FunctionRanges | None) -> bool:
"""Design §Scorer rule (1): the record lies inside the labeled function's line range. No range, no match (never name equality)."""
spans = spans_named((ranges or {}).get(record.path, ()), function)
return record.line is not None and any(start <= record.line <= end for start, end in spans)
def _label_unresolved(expected: ExpectedFinding, ranges: FunctionRanges | None) -> bool:
"""A labeled function that no parsed range of the vulnerable side names cannot be matched; count it as loss."""
if not expected.function:
return False
return not any(spans_named(spans, expected.function) for spans in (ranges or {}).values())
def _overlay_matches_expected(
expected: ExpectedFinding, detected: Sequence[OverlayRecord], language: str, ranges: FunctionRanges | None = None
) -> bool:
if is_weak_label(expected):
return False
alias_sinks = _sink_ids_by_rule().get((_fact_language(language), expected.rule_id or ""), frozenset())
for record in detected:
function_ok = True
if expected.function:
function_ok = _in_named_function(record, expected.function, ranges)
if not function_ok:
continue
if expected.rule_id and expected.rule_id in record.proposal_id:
return True
if expected.cwe and expected.cwe == record.cwe:
if alias_sinks and any(sink in alias_sinks for sink in record.sinks):
return True
if expected.sink and expected.sink in record.sinks:
if expected.sink and expected.sink.lower() != "unknown" and expected.sink in record.sinks:
return True
# CWE equality only counts inside a named function (Req 1.2, 1.3).
if expected.function and function_ok and expected.cwe and expected.cwe == record.cwe:
return True
return False
def _overlay_leaks(case: PairCase, promoted: Sequence[OverlayRecord]) -> tuple[str, ...]:
leaked: list[str] = []
for record in promoted:
if any(_overlay_matches_expected(expected, [record]) for expected in case.expected) or case.fix_policy == "silent":
leaked.append(record.proposal_id)
def _hunter_matches_expected(
expected: ExpectedFinding, finding: StaticFinding, ranges: dict[str, tuple[tuple[str, int, int], ...]]
) -> bool:
if is_weak_label(expected):
return False
text = f"{finding.title} {finding.rationale} {' '.join(finding.tags)}".lower()
if expected.function:
spans = spans_named(ranges.get(finding.path, ()), expected.function)
if finding.line is None or not any(start <= finding.line <= end for start, end in spans):
return False
if expected.cwe and expected.cwe.lower() in text:
return True
if expected.sink and expected.sink.lower() != "unknown" and expected.sink.lower() in text:
return True
return bool(expected.rule_id and expected.rule_id in finding.finding_id)
def _overlay_leaks(case: PairCase, detected_fix: Sequence[OverlayRecord], ranges: FunctionRanges | None = None) -> tuple[str, ...]:
if case.fix_policy == "silent":
return tuple(record.proposal_id for record in promoted)
return tuple(leaked)
return tuple(record.proposal_id for record in detected_fix)
return tuple(
record.proposal_id
for record in detected_fix
if any(_overlay_matches_expected(row, [record], case.language, ranges) for row in case.expected)
)
def _fix_leaks(case: PairCase, findings: list[StaticFinding]) -> tuple[StaticFinding, ...]:
@@ -438,6 +823,13 @@ def _fix_leaks(case: PairCase, findings: list[StaticFinding]) -> tuple[StaticFin
return tuple(leaked)
def _mechanisms(case: PairCase) -> tuple[str, ...]:
return tuple(sorted({row.mechanism for row in case.expected if row.mechanism}))
# --- metrics ---------------------------------------------------------------
def _metrics(outcomes: Sequence[PairOutcome]) -> PairCorpusMetrics:
pairs = len(outcomes)
detected = sum(1 for item in outcomes if item.detected_vuln)
@@ -460,3 +852,18 @@ def _metrics(outcomes: Sequence[PairOutcome]) -> PairCorpusMetrics:
labeled_recall=labeled_matched / labeled_expected if labeled_expected else 1.0,
youden=(detected / pairs if pairs else 1.0) - (1.0 - (silent / pairs if pairs else 1.0)),
)
def _loss(outcomes: Sequence[PairOutcome]) -> dict[str, dict[str, int]]:
loss: dict[str, dict[str, int]] = {}
for item in outcomes:
bucket = loss.setdefault(
item.slice,
{"parse_failed_files": 0, "unadjudicated_proposals": 0, "weak_labels": 0, "known_limit": 0, "unresolved_function_labels": 0},
)
bucket["parse_failed_files"] += item.parse_failed_vuln + item.parse_failed_fixed
bucket["unadjudicated_proposals"] += item.unadjudicated_vuln + item.unadjudicated_fixed
bucket["weak_labels"] += item.weak_labels
bucket["unresolved_function_labels"] += item.unresolved_labels
bucket["known_limit"] += 1 if item.known_limit else 0
return loss
src/openultrasast/semantic/overlay.py
+30 -9
View File
@@ -12,11 +12,15 @@ from ..policy import load_policy, resolve_severity
from ..preprocess import FileTarget
from .engines import joern_available, joern_flow
from .facts import FactLoadError, SemanticFacts, load_facts
from .taint import Demotion, FileFlow, TaintPath, UnadjudicatedFlow, analyze_file, merge_joern_paths
from .functions import FunctionRange, function_at, named_ranges_from_ir
from .ir import parse_file
from .taint import Demotion, FileFlow, TaintPath, UnadjudicatedFlow, analyze_ir, merge_joern_paths
DISPOSITIONS = frozenset({"promote", "demote", "unadjudicated", "coverage"})
UNADJUDICATED_REASONS = frozenset({"parse_failed", "language_unsupported", "flow_incomplete", "facts_unavailable"})
ALLOWED_EVIDENCE = frozenset({"static_corroboration", "suspicion"})
__all__ = ["OverlayRecord", "adjudicate", "function_at", "FunctionRange", "write_overlay", "finding_from_coverage"]
FORBIDDEN_EVIDENCE = frozenset({"crash_reproduced", "exploit_demonstrated", "patch_validated", "root_cause_explained"})
@@ -40,6 +44,7 @@ class OverlayRecord:
origin: str = "inventory"
engine: str = "none"
language: str = ""
function: str | None = None # enclosing function when the file parsed (additive; pair-corpus-honesty)
def __post_init__(self) -> None:
if self.disposition not in DISPOSITIONS:
@@ -76,13 +81,13 @@ def adjudicate(
)
for finding in findings
]
flows = _flows_for(root, targets, loaded, joern_extra)
flows, ranges = _flows_for(root, targets, loaded, joern_extra)
records: list[OverlayRecord] = []
claimed: set[tuple[str, int | None, str]] = set()
for finding in findings:
language = language_by_path.get(finding.path, "")
flow = flows.get(finding.path)
record = _record_for_finding(finding, language, flow)
record = _record_for_finding(finding, language, flow, function_at(ranges.get(finding.path, ()), finding.line))
records.append(record)
claimed.add((finding.path, finding.line, _sink_hint(finding)))
for path, flow in flows.items():
@@ -110,6 +115,7 @@ def adjudicate(
origin="overlay",
engine=flow.engine,
language=language,
function=function_at(ranges.get(path, ()), taint.sink_line),
)
)
claimed.add(key)
@@ -156,11 +162,17 @@ def _flows_for(
targets: Sequence[FileTarget],
facts: SemanticFacts,
joern_extra: Sequence[TaintPath] | None,
) -> dict[str, FileFlow | UnadjudicatedFlow]:
) -> tuple[dict[str, FileFlow | UnadjudicatedFlow], dict[str, tuple[FunctionRange, ...]]]:
flows: dict[str, FileFlow | UnadjudicatedFlow] = {}
ranges: dict[str, tuple[FunctionRange, ...]] = {}
for target in targets:
text = _read_text(root / target.path)
analyzed = analyze_file(target.path, text, target.language, facts)
ir = parse_file(target.path, text, target.language)
if not ir.parse_ok:
flows[target.path] = UnadjudicatedFlow(path=target.path, reason=ir.reason or "parse_failed", engine=ir.engine)
continue
ranges[target.path] = named_ranges_from_ir(ir, text)
analyzed: FileFlow | UnadjudicatedFlow = analyze_ir(ir, facts.for_language(target.language))
if isinstance(analyzed, FileFlow):
extra = tuple(joern_extra) if joern_extra is not None else _joern_paths(target.path, text, target.language)
if extra:
@@ -173,7 +185,7 @@ def _flows_for(
incomplete=analyzed.incomplete,
)
flows[target.path] = analyzed
return flows
return flows, ranges
def _joern_paths(path: str, text: str, language: str) -> tuple[TaintPath, ...]:
@@ -187,7 +199,12 @@ def _joern_paths(path: str, text: str, language: str) -> tuple[TaintPath, ...]:
return tuple(paths)
def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow | UnadjudicatedFlow | None) -> OverlayRecord:
def _record_for_finding(
finding: StaticFinding,
language: str,
flow: FileFlow | UnadjudicatedFlow | None,
function: str | None = None,
) -> OverlayRecord:
if flow is None:
return _unadjudicated(finding, language, "flow_incomplete", "none")
if isinstance(flow, UnadjudicatedFlow):
@@ -210,6 +227,7 @@ def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow |
dominating_fact=demotion.dominating_fact,
engine=flow.engine,
language=language,
function=function,
)
if taint is not None and (demotion is None or not demotion.sanitizers):
# Sanitizer on the same path would have produced a demotion with sanitizers.
@@ -226,6 +244,7 @@ def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow |
evidence_level="static_corroboration",
engine=flow.engine,
language=language,
function=function,
)
if demotion is not None:
return OverlayRecord(
@@ -242,8 +261,9 @@ def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow |
dominating_fact=demotion.dominating_fact,
engine=flow.engine,
language=language,
function=function,
)
return _unadjudicated(finding, language, "flow_incomplete", flow.engine)
return _unadjudicated(finding, language, "flow_incomplete", flow.engine, function)
def _matching_taint(finding: StaticFinding, flow: FileFlow) -> TaintPath | None:
@@ -286,7 +306,7 @@ def _cwe_of(finding: StaticFinding) -> str:
return ""
def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: str) -> OverlayRecord:
def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: str, function: str | None = None) -> OverlayRecord:
return OverlayRecord(
proposal_id=finding.finding_id,
path=finding.path,
@@ -300,6 +320,7 @@ def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: s
evidence_level="static_corroboration",
engine=engine,
language=language,
function=function,
)
tests/test_pair_corpus.py
+554
View File
@@ -3,6 +3,8 @@
import json
from pathlib import Path
import pytest
from openultrasast.benchmark import ExpectedFinding
from openultrasast.cli import main
from openultrasast.gate import LANGUAGE_MANIFESTS
@@ -245,3 +247,555 @@ def test_cli_vfc_json_includes_dual_scorers() -> None:
assert "overlay" in payload["scorers"]["vfc"]
assert "inventory" in payload["scorers"]["vfc"]
assert payload["per_slice"]["vfc"]["pairs"] == payload["scorers"]["vfc"]["overlay"]["pairs"]
# --- pair-corpus-honesty: honest scorer -------------------------------------
def _case(
tmp_path: Path, name: str, vuln_src: str, fixed_src: str, expected: ExpectedFinding, *, slice_name: str = "sast", **extra: object
) -> PairCase:
vuln = tmp_path / f"{name}-vuln.py"
fixed = tmp_path / f"{name}-fixed.py"
vuln.write_text(vuln_src)
fixed.write_text(fixed_src)
return PairCase(
name=name,
slice=slice_name,
language="python",
origin="test",
vuln_file=vuln,
fixed_file=fixed,
relpath="app.py",
expected=(expected,),
min_recall=1.0,
fix_policy="silent",
provenance=str(extra.get("provenance", "synthetic")),
split=str(extra.get("split", "train")),
known_limit=extra.get("known_limit"), # type: ignore[arg-type]
)
def test_coverage_with_source_counts_as_detection_and_as_leak(tmp_path: Path) -> None:
# hashlib.new is a fact sink the regex inventory does not spell -> overlay coverage, never promote.
src = "import hashlib\nfrom flask import request\n\ndef h():\n data = request.args.get('x')\n return hashlib.new(data)\n"
expected = ExpectedFinding(
cwe="CWE-327",
vulnerability_class="weak hash",
path="app.py",
evidence="",
rule_id="python-weak-hash",
sink="hashlib",
mechanism="config_sink_weak_literal",
)
both = evaluate_pair(_case(tmp_path, "cov", src, src, expected))
assert both.detected_vuln and "coverage" in both.detection_kinds
assert not both.silent_fix # the same coverage record on the fixed side is a leak
def test_function_scoped_match_rejects_detection_outside_the_named_function(tmp_path: Path) -> None:
src = "from flask import request\n\ndef other():\n return eval(request.args.get('x'))\n\ndef target():\n return 1\n"
row = ExpectedFinding(
cwe="CWE-95",
vulnerability_class="code injection",
path="app.py",
evidence="",
rule_id="python-unsafe-eval",
sink="eval",
function="target",
mechanism="source_reaches_sink",
)
outcome = evaluate_pair(_case(tmp_path, "fn", src, "def target():\n return 1\n", row))
assert not outcome.detected_vuln and outcome.silent_fix
row_ok = ExpectedFinding(**{**row.__dict__, "function": "other"})
assert evaluate_pair(_case(tmp_path, "fn2", src, "def target():\n return 1\n", row_ok)).detected_vuln
def test_cwe_only_row_is_a_weak_label_and_never_matches(tmp_path: Path) -> None:
src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n"
weak = ExpectedFinding(cwe="CWE-95", vulnerability_class="code injection", path="app.py", evidence="", mechanism="source_reaches_sink")
outcome = evaluate_pair(_case(tmp_path, "weak", src, "def f():\n return 1\n", weak))
assert outcome.weak_labels == 1
assert not outcome.detected_vuln
def test_catalog_rejects_unknown_mechanism_and_sink_unknown_without_anchor(tmp_path: Path) -> None:
from openultrasast.pairs import CatalogError, _load_catalog_file
(tmp_path / "v.py").write_text("x = 1\n")
(tmp_path / "f.py").write_text("x = 1\n")
head = (
'[[pair]]\nname = "t"\nslice = "sast"\nvuln = "v.py"\nfixed = "f.py"\nrelpath = "app.py"\n\n'
'[[pair.expected]]\ncwe = "CWE-95"\nclass = "x"\npath = "app.py"\n'
)
(tmp_path / "bad_mech.toml").write_text(head + 'sink = "eval"\nmechanism = "not_a_mechanism"\n')
with pytest.raises(CatalogError, match="unknown mechanism"):
_load_catalog_file(tmp_path / "bad_mech.toml")
(tmp_path / "bad_sink.toml").write_text(head + 'sink = "unknown"\nmechanism = "other"\n')
with pytest.raises(CatalogError, match="sink=unknown"):
_load_catalog_file(tmp_path / "bad_sink.toml")
(tmp_path / "bad_prov.toml").write_text(
head.replace('slice = "sast"\n', 'slice = "sast"\nprovenance = "robot"\n') + 'sink = "eval"\nmechanism = "other"\n'
)
with pytest.raises(CatalogError, match="unknown provenance"):
_load_catalog_file(tmp_path / "bad_prov.toml")
def test_known_limit_pairs_are_evaluated_but_excluded_from_achievable(tmp_path: Path) -> None:
src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n"
row = ExpectedFinding(
cwe="CWE-95",
vulnerability_class="code injection",
path="app.py",
evidence="",
rule_id="python-unsafe-eval",
sink="eval",
mechanism="cross_artifact",
)
limited = _case(tmp_path, "lim", src, "def f():\n return 1\n", row, known_limit="cross_artifact: needs a properties file")
plain = _case(
tmp_path,
"plain",
src,
"def f():\n return 1\n",
ExpectedFinding(**{**row.__dict__, "mechanism": "source_reaches_sink"}),
provenance="agent",
split="holdout",
)
result = evaluate_catalog([limited, plain])
assert result.known_limit == ("lim",)
assert result.per_slice["sast"].pairs == 2 and result.achievable["sast"].pairs == 1
assert set(result.per_profile) == {"synthetic", "agent"}
assert set(result.per_mechanism) == {"cross_artifact", "source_reaches_sink"}
assert result.loss["sast"]["known_limit"] == 1
assert "overlay" in result.scorers["sast"] and "inventory" in result.scorers["sast"]
assert result.degradations and result.degradations[0]["reason"] == "hunter_model_unavailable"
def test_hunter_scorer_uses_same_rules_with_a_scripted_client(tmp_path: Path) -> None:
from openultrasast.findings import StaticFinding
src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n"
row = ExpectedFinding(
cwe="CWE-95",
vulnerability_class="code injection",
path="app.py",
evidence="",
sink="eval",
function="f",
mechanism="source_reaches_sink",
)
def scripted(root: Path) -> list[StaticFinding]:
text = (root / "app.py").read_text()
if "eval(" not in text:
return []
return [
StaticFinding(
finding_id="tool-hunter:app.py:4",
path="app.py",
title="eval of request input",
severity="high",
confidence="medium",
evidence_level="suspicion",
rationale="CWE-95 eval executes attacker input",
line=4,
function_name=None,
reachability_status="unknown",
reachability_evidence=[],
reachability_conditions=[],
tags=[],
ranking_priority=1.0,
)
]
case = _case(tmp_path, "hunt", src, "def f():\n return 1\n", row)
result = evaluate_catalog([case], hunter=scripted)
hunter = result.scorers["sast"]["hunter"]
assert hunter.pair_correct == 1 and hunter.detected_vuln == 1 and hunter.silent_fix == 1
assert not result.degradations
def test_juliet_strcpy_pair_bodies_differ_and_java_hash_is_known_limit() -> None:
cases = {case.name: case for case in select_slice(load_pair_catalog(), "sast")}
strcpy = cases["juliet-c-cwe121-strcpy"]
assert strcpy.vuln_file.read_text() != strcpy.fixed_file.read_text()
assert "dataBadBuffer[10]" in strcpy.vuln_file.read_text() and "data = dataGoodBuffer" in strcpy.fixed_file.read_text()
assert cases["owasp-java-hash"].known_limit and cases["owasp-java-hash"].known_limit.startswith("cross_artifact")
assert all(row.mechanism for case in cases.values() for row in case.expected)
assert all(case.provenance == "synthetic" for case in cases.values())
def test_vfc_rows_name_functions_and_carry_no_weak_labels() -> None:
cases = select_slice(load_pair_catalog(), "vfc")
from openultrasast.pairs import is_weak_label
assert cases and all(row.function for case in cases for row in case.expected)
assert not any(is_weak_label(row) for case in cases for row in case.expected)
assert {case.split for case in cases} == {"train", "holdout"}
def test_cli_pairs_profile_and_split_filters() -> None:
import io
from contextlib import redirect_stdout
buf = io.StringIO()
with redirect_stdout(buf):
assert main(["pairs", "--slice", "sast", "--profile", "synthetic", "--split", "holdout", "--json"]) == 0
payload = json.loads(buf.getvalue())
assert payload["overall"]["pairs"] >= 3
assert set(payload["per_profile"]) == {"synthetic"}
assert "per_mechanism" in payload and "loss" in payload and "achievable" in payload
assert all(item["split"] == "holdout" for item in payload["outcomes"])
def test_new_slices_load_offline_and_score_with_profiles() -> None:
from openultrasast.pairs import PROVENANCES
catalog = load_pair_catalog()
# agent-vfc rows enter the loaded catalog only after human review (Req 5.3); candidates wait in a separate file.
candidates = Path("benchmarks/pairs/agent-vfc/catalog-candidates.toml")
assert candidates.is_file() and "reviewer" in candidates.read_text()
assert all(case.provenance == "agent" for case in select_slice(catalog, "agent-vfc"))
for slice_name, expected_provenance in (("vibe-py", "human"), ("vfc-js", "human")):
cases = select_slice(catalog, slice_name)
assert cases, slice_name
assert all(case.vuln_file.is_file() and case.fixed_file.is_file() for case in cases)
assert all(case.license for case in cases), slice_name
assert all(case.provenance == expected_provenance for case in cases), slice_name
assert {case.split for case in cases} <= {"train", "holdout"}
header = cases[0].vuln_file.read_text()[:600]
assert "Provenance:" in header and "license:" in header
result = evaluate_catalog(cases[:2])
assert result.per_slice[slice_name].pairs == 2
assert set(result.per_profile) <= PROVENANCES and expected_provenance in result.per_profile
assert result.per_mechanism and slice_name in result.loss
# --- review round 1 remediation (RED first) -----------------------------------
def test_weak_label_never_matches_on_inventory_fallback(tmp_path: Path) -> None:
# A syntax error forces parse_failed on both sides -> overlay falls back to inventory.
src = "from flask import request\n\ndef f(:\n return eval(request.args.get('x'))\n"
weak = ExpectedFinding(cwe="CWE-95", vulnerability_class="code injection", path="app.py", evidence="", mechanism="source_reaches_sink")
outcome = evaluate_pair(_case(tmp_path, "weakfb", src, "def f():\n return 1\n", weak))
assert outcome.parse_failed_vuln == 1
assert outcome.weak_labels == 1
assert not outcome.detected_vuln
assert outcome.misses == ("-:app.py:CWE-95",)
def test_sink_match_is_exact_or_alias_not_substring() -> None:
from openultrasast.pairs import _overlay_matches_expected
from openultrasast.semantic import OverlayRecord
record = OverlayRecord(
proposal_id="overlay-coverage:app.py:3:execute",
path="app.py",
line=3,
disposition="coverage",
reason="x",
cwe="CWE-89",
sources=("request",),
sinks=("execute",),
sanitizers=(),
evidence_level="static_corroboration",
)
exec_row = ExpectedFinding(
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", sink="exec", mechanism="source_reaches_sink"
)
assert not _overlay_matches_expected(exec_row, [record], "python")
exact = ExpectedFinding(
cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", sink="execute", mechanism="source_reaches_sink"
)
assert _overlay_matches_expected(exact, [record], "python")
alias = ExpectedFinding(
cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", rule_id="python-sql-injection", mechanism="source_reaches_sink"
)
assert _overlay_matches_expected(alias, [record], "python")
def test_vfc_style_slice_rows_require_a_named_function(tmp_path: Path) -> None:
from openultrasast.pairs import CatalogError, _load_catalog_file
(tmp_path / "v.js").write_text("x = 1\n")
(tmp_path / "f.js").write_text("x = 1\n")
head = (
'[[pair]]\nname = "t"\nslice = "vfc-js"\nvuln = "v.js"\nfixed = "f.js"\nrelpath = "a.js"\n\n'
'[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "a.js"\nmechanism = "source_reaches_sink"\n'
)
(tmp_path / "nofn.toml").write_text(head + 'sink = "exec"\n')
with pytest.raises(CatalogError, match="function"):
_load_catalog_file(tmp_path / "nofn.toml")
(tmp_path / "anon.toml").write_text(head + 'function = "<anon>"\n')
with pytest.raises(CatalogError, match="function"):
_load_catalog_file(tmp_path / "anon.toml")
(tmp_path / "ok.toml").write_text(head + 'function = "run"\n')
assert _load_catalog_file(tmp_path / "ok.toml")[0].expected[0].function == "run"
def test_known_limit_pairs_emit_no_improve_signals(tmp_path: Path) -> None:
src = "def f():\n return 1\n"
row = ExpectedFinding(
cwe="CWE-95",
vulnerability_class="x",
path="app.py",
evidence="",
rule_id="python-unsafe-eval",
sink="eval",
mechanism="cross_artifact",
)
limited = _case(tmp_path, "lim2", src, src, row, known_limit="cross_artifact: needs a properties file")
result = evaluate_catalog([limited])
assert not result.outcomes[0].detected_vuln
assert all(signal["pair"] != "lim2" for signal in result.signals)
# --- review round 2 remediation (RED first) -----------------------------------
def test_function_at_skips_anonymous_and_module_ranges() -> None:
from openultrasast.semantic.overlay import function_at
ranges = (("<module>", 1, 40), ("handler", 3, 20), ("<anon>", 5, 12))
assert function_at(ranges, 7) == "handler" # innermost *named* function, never "<anon>"
assert function_at(ranges, 25) is None
assert function_at((("<anon>", 1, 9),), 4) is None
@pytest.mark.semantic
def test_promotion_inside_a_callback_matches_the_enclosing_named_function(tmp_path: Path) -> None:
from openultrasast.semantic.extra import has_semantic_extra
if not has_semantic_extra():
pytest.skip("tree-sitter extra absent")
src = (
"const cp = require('child_process');\n"
"function handler(req, res) {\n"
" [req.query.cmd].forEach(function (c) {\n"
" cp.exec(c);\n"
" });\n"
"}\n"
)
fixed = "function handler(req, res) {\n res.end('ok');\n}\n"
vuln = tmp_path / "v.js"
fix = tmp_path / "f.js"
vuln.write_text(src)
fix.write_text(fixed)
row = ExpectedFinding(
cwe="CWE-78", vulnerability_class="x", path="app.js", evidence="", function="handler", sink="exec", mechanism="source_reaches_sink"
)
case = PairCase(
name="cb",
slice="vfc-js",
language="javascript",
origin="test",
vuln_file=vuln,
fixed_file=fix,
relpath="app.js",
expected=(row,),
min_recall=1.0,
fix_policy="silent",
provenance="human",
)
outcome = evaluate_pair(case)
assert outcome.detected_vuln, outcome
assert outcome.pair_correct
def test_loader_rejects_reserved_words_as_function_labels(tmp_path: Path) -> None:
from openultrasast.pairs import CatalogError, _load_catalog_file
(tmp_path / "v.js").write_text("x = 1\n")
(tmp_path / "f.js").write_text("x = 1\n")
head = (
'[[pair]]\nname = "t"\nslice = "vfc-js"\nvuln = "v.js"\nfixed = "f.js"\nrelpath = "a.js"\n\n'
'[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "a.js"\nmechanism = "source_reaches_sink"\nsink = "exec"\n'
)
for bad in ("function", "def", "return"):
(tmp_path / "bad.toml").write_text(head + f'function = "{bad}"\n')
with pytest.raises(CatalogError, match="function"):
_load_catalog_file(tmp_path / "bad.toml")
def test_agent_vfc_stays_empty_until_reviewed() -> None:
catalog = load_pair_catalog()
loaded = select_slice(catalog, "agent-vfc")
candidates = Path("benchmarks/pairs/agent-vfc/catalog-candidates.toml")
# Req 5.3: nothing loads until a human sets `reviewer`; the candidates file must not be empty meanwhile.
assert candidates.is_file() and candidates.read_text().count("[[pair]]") >= 1
assert loaded == () or all(case.provenance == "agent" for case in loaded)
reviewed_in_recipes = 'reviewer = "pending"' not in Path("benchmarks/pairs/agent-vfc/recipes.toml").read_text()
assert bool(loaded) == reviewed_in_recipes
# --- debug round after review 3: shared named ranges, containment-only matching -----
def _js_case(tmp_path: Path, name: str, src: str, function: str) -> PairCase:
vuln = tmp_path / f"{name}-v.js"
fix = tmp_path / f"{name}-f.js"
vuln.write_text(src)
fix.write_text("function untouched() {\n return 1;\n}\n")
row = ExpectedFinding(
cwe="CWE-78", vulnerability_class="x", path="app.js", evidence="", function=function, sink="exec", mechanism="source_reaches_sink"
)
return PairCase(
name=name,
slice="vfc-js",
language="javascript",
origin="test",
vuln_file=vuln,
fixed_file=fix,
relpath="app.js",
expected=(row,),
min_recall=1.0,
fix_policy="silent",
provenance="human",
)
_JS_FORMS = {
"arrow": "const cp = require('child_process');\nconst h = (req, res) => {\n cp.exec(req.query.cmd);\n};\n",
"exports": "const cp = require('child_process');\nexports.h = function (req, res) {\n cp.exec(req.query.cmd);\n};\n",
"method": "const cp = require('child_process');\nclass Ctl {\n h(req, res) {\n cp.exec(req.query.cmd);\n }\n}\n",
"property": (
"const cp = require('child_process');\nmodule.exports = {\n h: function (req, res) {\n cp.exec(req.query.cmd);\n },\n};\n"
),
}
@pytest.mark.semantic
@pytest.mark.parametrize("form", sorted(_JS_FORMS))
def test_declarator_named_functions_match_their_labels(tmp_path: Path, form: str) -> None:
from openultrasast.semantic.extra import has_semantic_extra
if not has_semantic_extra():
pytest.skip("tree-sitter extra absent")
outcome = evaluate_pair(_js_case(tmp_path, form, _JS_FORMS[form], "h"))
assert outcome.detected_vuln, (form, outcome)
@pytest.mark.semantic
def test_containment_matches_outer_label_and_rejects_inner_label(tmp_path: Path) -> None:
from openultrasast.semantic.extra import has_semantic_extra
if not has_semantic_extra():
pytest.skip("tree-sitter extra absent")
src = (
"const cp = require('child_process');\n"
"function outer(req) {\n"
" function inner(c) {\n"
" cp.exec(c);\n"
" }\n"
" inner(req.query.cmd);\n"
" cp.exec(req.query.top);\n"
"}\n"
)
# The sink inside `inner` also lies inside `outer`'s range: containment says yes, name equality would say no.
assert evaluate_pair(_js_case(tmp_path, "outer", src, "outer")).detected_vuln
# A label naming `inner` must not match the promote at line 7, which is outside `inner`.
only_top = src.replace(" cp.exec(c);\n", " return c;\n")
assert not evaluate_pair(_js_case(tmp_path, "inner", only_top, "inner")).detected_vuln
def test_unresolvable_function_label_is_a_visible_loss_not_a_silent_miss(tmp_path: Path) -> None:
src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n"
row = ExpectedFinding(
cwe="CWE-95",
vulnerability_class="x",
path="app.py",
evidence="",
function="does_not_exist",
sink="eval",
mechanism="source_reaches_sink",
)
outcome = evaluate_pair(_case(tmp_path, "unres", src, "def f():\n return 1\n", row))
assert not outcome.detected_vuln
assert outcome.unresolved_labels == 1
assert any(miss.endswith("!unresolved") for miss in outcome.misses)
result = evaluate_catalog([_case(tmp_path, "unres2", src, "def f():\n return 1\n", row)])
assert result.loss["sast"]["unresolved_function_labels"] == 1
def test_named_function_ranges_use_declarator_names() -> None:
from openultrasast.semantic.extra import has_semantic_extra
from openultrasast.semantic.functions import declarator_name, named_function_ranges
assert declarator_name("const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {") == "listProcessesOnPort"
assert declarator_name(" exec('df -k ' + path, function(err, stdout) {") is None
assert declarator_name(" function (req, res) {") is None
if not has_semantic_extra():
pytest.skip("tree-sitter extra absent")
src = "\n".join(
[
"function a() { return 1; }",
"const b = (x) => { return x; };",
"exports.c = function (y) { return y; };",
"class K {",
" d(z) {",
" return z;",
" }",
"}",
"const o = {",
" e: function (w) {",
" return w;",
" },",
"};",
"",
]
)
ranges = named_function_ranges("f.js", src, "javascript")
assert ranges is not None
assert {name for name, _, _ in ranges} >= {"a", "b", "c", "d", "e"}
assert "<anon>" not in {name for name, _, _ in ranges}
# --- review round 4 remediation (RED first) -----------------------------------
@pytest.mark.semantic
def test_c_function_with_return_type_on_previous_line_keeps_its_parser_name() -> None:
from openultrasast.semantic.extra import has_semantic_extra
from openultrasast.semantic.functions import function_at, named_function_ranges
if not has_semantic_extra():
pytest.skip("tree-sitter extra absent")
ranges = named_function_ranges("a.c", "static int\nfoo(int x)\n{\n return x;\n}\n", "c")
assert ranges is not None and any(name == "foo" and start <= 4 <= end for name, start, end in ranges)
assert function_at(ranges, 4) == "foo"
def test_declarator_name_knows_perl_subs_and_never_names_calls() -> None:
from openultrasast.semantic.functions import declarator_name
assert declarator_name("sub link_hash_cert {") == "link_hash_cert"
assert declarator_name(" return foo(bar)") is None
assert declarator_name(" print(x)") is None
assert declarator_name("synchronized (lock) {") is None
assert declarator_name("int main(int argc, char **argv)") == "main"
# --- review round 5 remediation (RED first) -----------------------------------
def test_declarator_name_strips_cpp_qualification_to_match_unqualified_labels() -> None:
from openultrasast.semantic.functions import declarator_name
assert declarator_name("bool FileReaderLoader::ArrayBufferResult(int a) {") == "ArrayBufferResult"
assert declarator_name("void ns::Klass::run(const std::string& s) const {") == "run"
assert declarator_name("std::string Foo::bar() {") == "bar"
@pytest.mark.semantic
def test_cpp_qualified_method_range_is_found_by_its_unqualified_label() -> None:
from openultrasast.semantic.extra import has_semantic_extra
from openultrasast.semantic.functions import function_at, named_function_ranges, spans_named
if not has_semantic_extra():
pytest.skip("tree-sitter extra absent")
ranges = named_function_ranges("a.cc", "bool K::m(int a) {\n return a;\n}\n", "cpp")
assert ranges is not None
assert spans_named(ranges, "m") == ((1, 3),)
assert function_at(ranges, 2) == "m"