mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 20:19:29 +00:00
Marius CiepluchandClaude Opus 5.5
602ec23e03
The answer check counted only the rendered function text as shown, so 7 of 8 baseline suspicions citing lines the model had read with read_file were marked unknown (baseline 1/12 against the pilot hunter's 6/12). Tools now record the lines they return. A DSPy parse failure scores as zero findings with a flag instead of a broken run. The per-run response cache now takes effect through dspy.configure_cache. Co-Authored-By: Claude Opus 5.5 <[email protected]>
38 lines
1.6 KiB
Python
38 lines
1.6 KiB
Python
"""Attach benchmark fix-site labels without exposing them to feature producers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
from copy import deepcopy
|
|
from typing import Any
|
|
|
|
|
|
def label_rows(rows: Sequence[Mapping[str, Any]], change: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|
"""Label verified sites, leave other positive rows unknown and label ordinary rows zero."""
|
|
kind = change.get("kind")
|
|
if kind not in ("vulnerability", "ordinary"):
|
|
raise ValueError("change kind must be vulnerability or ordinary")
|
|
if not change.get("case_id"):
|
|
raise ValueError("a label needs a traceable case_id")
|
|
if kind == "vulnerability" and not all(change.get(k) for k in ("path", "function", "function_lines", "family")):
|
|
raise ValueError("a vulnerability label needs its path, function, span and family")
|
|
result = []
|
|
for row in rows:
|
|
data = deepcopy(dict(row))
|
|
span = change.get("function_lines")
|
|
function = row.get("function", row.get("feature.source.function"))
|
|
site = (
|
|
kind == "vulnerability"
|
|
and row.get("path") == change["path"]
|
|
and (function == change["function"] or (span[0] <= row.get("line", 0) <= span[1]))
|
|
)
|
|
data.update(
|
|
label=1 if site else 0 if kind == "ordinary" else None,
|
|
family=change["family"] if site else None,
|
|
fix_span=list(span) if site else None,
|
|
label_ref=change["case_id"] if site or kind == "ordinary" else None,
|
|
verification=None,
|
|
)
|
|
result.append(data)
|
|
return result
|