marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-20 13:39:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pair-corpus-honesty): review tiers on every pair, agent-vfc loads at tier title

Browse Source 
Tasks 8.1, 7.1 (tests) and the re-scoped 5.3.

- pairs.py: review_tier in {seeded, advisory, title, reviewed} with
  per-slice defaults (local reviewed, vibe-py seeded, sast/vfc/vfc-js/
  github advisory, agent-vfc title); reviewer = "pending" means title, a
  named reviewer means reviewed, reviewed without a reviewer is rejected;
  per_tier metrics in the payload; select_tier and GATING_TIERS for the
  improve loop.
- catalog_gen.py: stamps review_tier (and reviewer) on every generated
  row and training manifest row; title rows enter catalog.toml; the
  catalog-candidates.toml queue is retired; precedence matches the loader.
- agent-vfc: 29 title-tier rows load and score (0/29, Youden 0.0, the
  honest unreviewed baseline); local rows name their reviewer.
- Tests: tier defaults, loader rejections, per_tier payload, the Req 8.1
  catalog-on-disk / stage-1 exclusion proof and per-slice CLI checks.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-05 10:45:43 +02:00
co-authored by Claude Fable 5.1
parent 9e7aef6a6b
commit b1c5f36076
18 changed files with 1236 additions and 967 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/pairs/vibe-py/catalog.toml
+35
View File
@@ -22,6 +22,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-78"
@@ -50,6 +51,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -78,6 +80,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-78"
@@ -106,6 +109,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -134,6 +138,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-502"
@@ -162,6 +167,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -190,6 +196,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -218,6 +225,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-326"
@@ -246,6 +254,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-78"
@@ -274,6 +283,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -302,6 +312,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-16"
@@ -330,6 +341,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-1004"
@@ -358,6 +370,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -386,6 +399,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-502"
@@ -414,6 +428,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -442,6 +457,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -470,6 +486,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-1336"
@@ -498,6 +515,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-79"
@@ -526,6 +544,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-862"
@@ -554,6 +573,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-639"
@@ -582,6 +602,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -610,6 +631,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-639"
@@ -638,6 +660,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -666,6 +689,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-639"
@@ -694,6 +718,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -722,6 +747,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -750,6 +776,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -778,6 +805,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-79"
@@ -806,6 +834,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-22"
@@ -834,6 +863,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-78"
@@ -862,6 +892,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-78"
@@ -890,6 +921,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-502"
@@ -918,6 +950,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -946,6 +979,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"
@@ -974,6 +1008,7 @@ min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
review_tier = "seeded"
[[pair.expected]]
cwe = "CWE-89"