marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 20:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(learning-harness): fix dates and the post-cutoff slice

Browse Source 
The catalog generator fills `fix_date` from the recipe at the precision the
recipe has -- a full ISO day when one was recorded, otherwise the year.
Every harvested recipe carries a year and almost none carries a day, and
writing `-01-01` would turn a year into a claim about a month. A row counts
as post-cutoff only when its earliest possible date is after the cutoff, so
a coarse date can under-claim the clean slice but never over-claim it.

Each family now reports the same numbers over the rows fixed after the
cutoff, beside the whole number and never instead of it, with the undated
count on its own line. Without a configured cutoff every scoreboard says so
rather than inventing one: guessing a model's training cutoff and then
publishing a contamination number from the guess would be worse than
reporting none.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 13:15:20 +02:00
co-authored by Claude Opus 5
parent d1faa04771
commit 3e71e9e88b
12 changed files with 1195 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/pairs/agent-vfc/catalog.toml
+59
View File
@@ -25,6 +25,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -56,6 +57,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-89"
@@ -85,6 +87,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-918"
@@ -114,6 +117,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-918"
@@ -145,6 +149,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -176,6 +181,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -208,6 +214,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-862"
@@ -239,6 +246,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -268,6 +276,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -299,6 +308,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-639"
@@ -330,6 +340,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -361,6 +372,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -390,6 +402,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -421,6 +434,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-862"
@@ -450,6 +464,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -481,6 +496,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -513,6 +529,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-89"
@@ -542,6 +559,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -571,6 +589,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -602,6 +621,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -633,6 +653,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -664,6 +685,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -695,6 +717,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-798"
@@ -726,6 +749,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -758,6 +782,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -789,6 +814,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -820,6 +846,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-89"
@@ -852,6 +879,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -882,6 +910,7 @@ provenance = "agent"
split = "holdout"
review_tier = "title"
known_limit = "client_side_caller"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-862"
@@ -912,6 +941,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -941,6 +971,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -970,6 +1001,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -999,6 +1031,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -1028,6 +1061,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -1059,6 +1093,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -1088,6 +1123,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -1117,6 +1153,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-918"
@@ -1148,6 +1185,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-798"
@@ -1179,6 +1217,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -1209,6 +1248,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-639"
@@ -1239,6 +1279,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-639"
@@ -1269,6 +1310,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.context]]
relpath = "services/web/app/src/router.mjs"
@@ -1304,6 +1346,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -1333,6 +1376,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-1321"
@@ -1363,6 +1407,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -1394,6 +1439,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-89"
@@ -1426,6 +1472,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2025"
[[pair.expected]]
cwe = "CWE-601"
@@ -1457,6 +1504,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-918"
@@ -1486,6 +1534,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-639"
@@ -1516,6 +1565,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-639"
@@ -1548,6 +1598,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-601"
@@ -1577,6 +1628,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-78"
@@ -1608,6 +1660,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-639"
@@ -1637,6 +1690,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -1668,6 +1722,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"
@@ -1697,6 +1752,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-22"
@@ -1726,6 +1782,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-918"
@@ -1755,6 +1812,7 @@ fix_policy = "silent"
provenance = "agent"
split = "holdout"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-89"
@@ -1784,6 +1842,7 @@ fix_policy = "silent"
provenance = "agent"
split = "train"
review_tier = "title"
fix_date = "2026"
[[pair.expected]]
cwe = "CWE-16"