marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 11:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(learning-harness): the round-4 review findings

Browse Source 
Three defects, two of them the same shape as ones already rejected once:

- The meta-agent proposer still could not propose. `MetaAgent(inner_model=)`
  takes a ModelConfig; it was handed a model name, a `# type: ignore` hid
  that from mypy, and a bare `except Exception` turned the resulting
  AttributeError into `meta_agent_failed` -- which reads exactly like the
  model declining. The composition is now a seam a test can pin, and a
  construction error is re-raised rather than reported as a quiet round.
- `ousast pairs --hunter` crashed in its documented form: the K floor landed
  with `--k-runs` still defaulting to 1. The covering test always passed
  `--k-runs 4`, which is why nothing noticed.
- 28 rows (12% of the corpus) resolved a span on the vulnerable side and
  none on the fixed side, so every fixed-side finding read as outside the
  labeled function and the pair scored as silence whatever the detector
  said. `fix_function` now reaches the catalog and the scorer, and the
  generator derives it from the fixed excerpt for the 25 trap rows that
  never declared one. Measured after: 0 rows.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 14:03:10 +02:00
co-authored by Claude Opus 5
parent 75ae9ee47b
commit 2dad25d312
17 changed files with 427 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/pairs/catalog_gen.py
+28
View File
@@ -93,6 +93,9 @@ def catalog_text(slice_name: str, recipes: list[dict[str, object]]) -> str:
lines.append(f'reviewer = "{_q(reviewer)}"')
if recipe.get("known_limit"):
lines.append(f'known_limit = "{_q(recipe["known_limit"])}"')
if recipe.get("fix_function"):
# Req 3.2: the fixed side of a trap pair is a different handler, and the scorer has to know which one
lines.append(f'fix_function = "{_q(recipe["fix_function"])}"')
fix_date = _fix_date(recipe)
if fix_date:
lines.append(f'fix_date = "{_q(fix_date)}"') # Req 10.5: the post-cutoff slice needs a date to sort by
@@ -399,6 +402,27 @@ def _stamp_function(path: Path, function: str) -> None:
path.write_text(updated)
def derive_fix_function(slice_root: Path, recipe: dict[str, object]) -> str:
"""The function the fixed excerpt actually holds, when it is not the labeled one.
Real-Vuln-Benchmark traps take their fixed side from a different, correctly guarded handler of the same
snapshot. Without this the fixed side resolves no span, every finding there counts as outside the labeled
function, and the pair scores as silence instead of as a leak — always in the direction that flatters the
number (learning-harness Req 3.2)."""
_vuln, fixed = excerpt_rel(recipe)
path = slice_root / fixed
if not path.is_file():
return ""
ranges = _ranges(path, _language(recipe))
if not ranges:
return ""
labeled = str(recipe.get("function", "") or "")
names = [name for name, _start, _end in ranges]
if labeled in names or len(set(names)) != 1:
return "" # the fixed side holds the labeled function, or holds several and the recipe must say which
return names[0]
def generate(slice_name: str, *, prune: bool = False) -> tuple[Path, int]:
slice_root = ROOT / slice_name
loaded = [dict(item) for item in tomllib.loads((slice_root / "recipes.toml").read_text()).get("recipe", [])]
@@ -419,6 +443,10 @@ def generate(slice_name: str, *, prune: bool = False) -> tuple[Path, int]:
recipe["function"] = derived
_stamp_function(slice_root / vuln, derived)
_stamp_function(slice_root / fixed, derived)
if not recipe.get("fix_function"):
derived = derive_fix_function(slice_root, recipe)
if derived:
recipe["fix_function"] = derived
recipes.append(recipe) # `reviewer = "pending"` rows load at tier `title` (Req 9.4); no separate queue file
else:
print(f"skip {recipe['name']}: excerpt missing (harvest failed or not fetched)")
benchmarks/pairs/vfc/catalog.toml
+1
View File
@@ -492,6 +492,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "advisory"
fix_function = "OSSL_TRACE_BEGIN"
fix_date = "2025"
[[pair.expected]]
benchmarks/pairs/vibe-py/catalog.toml
+28
View File
@@ -2153,6 +2153,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "mutate"
fix_date = "2026"
[[pair.expected]]
@@ -2183,6 +2184,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "mutate"
fix_date = "2026"
[[pair.expected]]
@@ -2213,6 +2215,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "parse_field"
fix_date = "2026"
[[pair.expected]]
@@ -2243,6 +2246,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "forgot_password"
fix_date = "2026"
[[pair.expected]]
@@ -2273,6 +2277,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "parse_field"
fix_date = "2026"
[[pair.expected]]
@@ -2333,6 +2338,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "run"
fix_date = "2026"
[[pair.expected]]
@@ -2363,6 +2369,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "run"
fix_date = "2026"
[[pair.expected]]
@@ -2424,6 +2431,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "get_by_username"
fix_date = "2026"
[[pair.expected]]
@@ -2454,6 +2462,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "get_by_username"
fix_date = "2026"
[[pair.expected]]
@@ -2484,6 +2493,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "get_by_username"
fix_date = "2026"
[[pair.expected]]
@@ -2514,6 +2524,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "do_login"
fix_date = "2026"
[[pair.expected]]
@@ -2544,6 +2555,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "do_login"
fix_date = "2026"
[[pair.expected]]
@@ -2574,6 +2586,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "do_signup"
fix_date = "2026"
[[pair.expected]]
@@ -3366,6 +3379,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "post"
fix_date = "2026"
[[pair.expected]]
@@ -3397,6 +3411,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "post"
fix_date = "2026"
[[pair.expected]]
@@ -3428,6 +3443,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "login"
fix_date = "2026"
[[pair.expected]]
@@ -3495,6 +3511,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "register_user"
fix_date = "2026"
[[pair.expected]]
@@ -3525,6 +3542,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "register_user"
fix_date = "2026"
[[pair.context]]
@@ -3561,6 +3579,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "nosql_return_users_from_username"
fix_date = "2026"
[[pair.expected]]
@@ -3591,6 +3610,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "nosql_return_users_from_username"
fix_date = "2026"
[[pair.expected]]
@@ -3621,6 +3641,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "nosql_return_users_from_username"
fix_date = "2026"
[[pair.expected]]
@@ -3651,6 +3672,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "get_post"
fix_date = "2026"
[[pair.expected]]
@@ -3681,6 +3703,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "get_post"
fix_date = "2026"
[[pair.expected]]
@@ -3711,6 +3734,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "_execute"
fix_date = "2026"
[[pair.expected]]
@@ -3741,6 +3765,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "_execute"
fix_date = "2026"
[[pair.expected]]
@@ -3802,6 +3827,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "get_posts"
fix_date = "2026"
[[pair.expected]]
@@ -3832,6 +3858,7 @@ fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
fix_function = "get_posts"
fix_date = "2026"
[[pair.expected]]
@@ -3862,6 +3889,7 @@ fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
fix_function = "get_posts"
fix_date = "2026"
[[pair.expected]]