marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

learn(exp-003): decision-rule experiments and the precision-bound BLOCK arm

Browse Source 
kind: rule experiments score once from the response cache behind a
zero-ceiling meter (a miss is excluded and counted, never asked) and apply
each arm's BLOCK rule to the same held-out scores. Arm B chooses a per-family
raw-score threshold on the other outer folds only (lowest t with Wilson 95%
LB >= 0.95); a test fails if a fold's own labels move its threshold.
Manifest and frozen units (353 units, 146 groups, 244 paired) for exp-003.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-03 08:02:51 +02:00
co-authored by Claude Opus 5.5
parent 432fd2947e
commit dd7ebc119b
6 changed files with 1083 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plane/experiments/exp-003-precision-bound-blocking.units.jsonl
+353
View File
@@ -0,0 +1,353 @@
{"family": "deserialization", "fold": "outer-1", "group": "aio-libs/aiohttp", "label": 1, "pair": "ee577ee4b6076fab", "unit": "2523f55d035d1a772a842090bc335e5d755b2fe15b376dd5b6b9e30b9dae82d1"}
{"family": "deserialization", "fold": "outer-1", "group": "aio-libs/aiohttp", "label": 0, "pair": "ee577ee4b6076fab", "unit": "ca0f6c160d7839204f6fcc1709bcbc749a02a0b3b8bb648e2a238eacbf22ed78"}
{"family": "access_control", "fold": "outer-0", "group": "alchemycms/alchemy_cms", "label": 1, "pair": "c95b6b01d0c2db2a", "unit": "15540c23be0c42472a49039658ec2ec0bbd8c6b318e35dd040a9d427370e0ef0"}
{"family": "access_control", "fold": "outer-0", "group": "alchemycms/alchemy_cms", "label": 0, "pair": "c95b6b01d0c2db2a", "unit": "542471980417e964182d7d3db59a05fe12bdd072449f3bd77f7daa5515dbc909"}
{"family": "access_control", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "bcc636b23dfb5f570d8864b17fe07b6b4fb52b65a40988c32011f1422f74d504"}
{"family": "access_control", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "dbcf1e67a1be7ccf7d51701f9b8339339164055135443044c23a446b16b3d934"}
{"family": "injection", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "50408f68114ecb4de3bebd836d52d9afad86b22dd1843641ac8a49b617c09e33"}
{"family": "injection", "fold": "outer-0", "group": "anxolerd/dvpwa", "label": 1, "pair": "", "unit": "cafb0f00f80dc6beeba40eb95c764bf9a3569193070ba321c28677dfa425ec88"}
{"family": "output_encoding", "fold": "outer-3", "group": "apache/logging-log4net", "label": 1, "pair": "fe0e104d111c2555", "unit": "088036fab9962be2b96f9162ea4f32190575678d3c5ba96bdfeac178bc22b2c4"}
{"family": "output_encoding", "fold": "outer-3", "group": "apache/logging-log4net", "label": 0, "pair": "fe0e104d111c2555", "unit": "7fd2b56a816e25358477eee64487fb2c77f34c652703ad0a7120c368492017a3"}
{"family": "path", "fold": "outer-0", "group": "apache/wicket", "label": 1, "pair": "b6742570f7f0e026", "unit": "7c6c1d27aee6557f1b6bf371b8209003f0d2d1122cfa35b6629eb43f38769cf4"}
{"family": "path", "fold": "outer-0", "group": "apache/wicket", "label": 0, "pair": "b6742570f7f0e026", "unit": "df8300d84dd784d0be46969ea432ef97b8ce611bae7a896ed4f349bd4d6897db"}
{"family": "access_control", "fold": "outer-0", "group": "apolloconfig/apollo", "label": 1, "pair": "0f584bb97a154488", "unit": "222a84ffc7c58d45e8f10df221762b23bf20d0287ecd20f375558134928886ca"}
{"family": "access_control", "fold": "outer-0", "group": "apolloconfig/apollo", "label": 0, "pair": "0f584bb97a154488", "unit": "25e4c9d2941f59f072385a7bde3fd0bd76f9e986ed24b06ce81962ab447f5094"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "archivy/archivy", "label": 0, "pair": "bb3d7baed62e6bb7", "unit": "e8ae88fd2b677e3f64ac94bddb012e3b903afa294ecc87774d10087c0d50489c"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "archivy/archivy", "label": 1, "pair": "bb3d7baed62e6bb7", "unit": "f6b0d241891dc1d897d74d98b6ff7ada2c1f1ce7874ca32097630c06ca53e389"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "axllent/mailpit", "label": 1, "pair": "143b900b99a46cf4", "unit": "200f96cd784ff5c39d8bba22883f22bc8a7c87c6970a7c871802f48a45548337"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "axllent/mailpit", "label": 0, "pair": "143b900b99a46cf4", "unit": "7a0d62ec424849ec4681f4333fbb077e8239054d1e964bd99b40847ad7ccdb01"}
{"family": "injection", "fold": "outer-2", "group": "browserify/static-eval", "label": 1, "pair": "c12dbc8a50f7fe9b", "unit": "04beb8a0fd054b92c2dd0ed83907330fdee6124d86114df948ac72a297a7d51f"}
{"family": "injection", "fold": "outer-2", "group": "browserify/static-eval", "label": 0, "pair": "c12dbc8a50f7fe9b", "unit": "c5029016a6fd2e3f9df59dba0ce3c173bbbb0d2b1de22d151086e01e1b272119"}
{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "0e213aeabacf4205db024ea3c557e720c3aba911424555784917e967d6404867"}
{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "e8bc65415bd42e43", "unit": "4920230ef773595a9fd9a92a090dff42b48499a1dc5fc8831f941dae55cb2538"}
{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 1, "pair": "e8bc65415bd42e43", "unit": "49209a810f207eb61382e2ceeab7076936838a5652546f21c187d066e220160b"}
{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "9e2964884cdf281fa3f18ee3481aa62f9a8a0a195d605da2df5bd71876fe311a"}
{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 1, "pair": "", "unit": "f9cc6f1bbcd59f5dc0de8f96248f722ea0679f8afa47c53f70832151ce7616d3"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 0, "pair": "e69463f945f9a0f4", "unit": "5feace7b28e260175f80c682e9009025aea6fae3dc75409cabbb955b560d0372"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 1, "pair": "e69463f945f9a0f4", "unit": "799a4f5a5b9324e86c90752f0239f059984f44e8ff1ec0dd518d0d9413e34c66"}
{"family": "deserialization", "fold": "outer-2", "group": "cesargb/laravel-magiclink", "label": 0, "pair": "768fc9db8a75a351", "unit": "adc6527469618f72b1dd7776814717cbd0056a0c625e05bef74199171ce73f1a"}
{"family": "deserialization", "fold": "outer-2", "group": "cesargb/laravel-magiclink", "label": 1, "pair": "768fc9db8a75a351", "unit": "da47663a57e370a75cc65b810b8a380aa26d12dcabb1d9b3c02641398ff7d348"}
{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 0, "pair": "fab4ce020049724b", "unit": "b9be90242be627aa46462143ac85940a906232ebc0b47776b5729475b27d215e"}
{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 1, "pair": "fab4ce020049724b", "unit": "ece12ea7d87b4a12c4da30faed9eb530b51fc879c271b68ba5304b230cda8e79"}
{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 1, "pair": "9126ba83905eb8b4", "unit": "d8dc50b7bb40bbcfc3f3d4f1f8d3a4d17eb1c662d5f759efd18656a482dd2e32"}
{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 0, "pair": "9126ba83905eb8b4", "unit": "f426f9307621b3f2d6a8d1669cdb4c19e962b94e34fbb394d4660cbde5921c88"}
{"family": "deserialization", "fold": "outer-3", "group": "commenthol/serialize-to-js", "label": 0, "pair": "7ac5e21d9d8f4c47", "unit": "1d1efa2fd7323c264290d4d314c957c853e1c7a9c877a54d6f03ab7d2af2ca86"}
{"family": "deserialization", "fold": "outer-3", "group": "commenthol/serialize-to-js", "label": 1, "pair": "7ac5e21d9d8f4c47", "unit": "b5dc78c7ac0b76803dc98904142c1139083fb9daafb5bff40e7b5c3f14ee03f5"}
{"family": "deserialization", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "0894a7212280bab755cd9d93b1f0ede2c51e4fca76b76e5ed9e24ef8f8cf0b9c"}
{"family": "injection", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "02694b5fb85e9e52bf5eaaab4e0b628f1f44ff3a8d5ad026cc10afdbc4222eaf"}
{"family": "injection", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "db9a3e888780501dd3c2c125af2f9d47f68ebf02f0e0db8370b2ea09c6f9d070"}
{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9af96b099dedcc4005e5fcd19ad78f73864e1cfbafbe6953b4edb6260957943d"}
{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9e7c2a786068a936a6c664cc229d6f39cea1fe57caf0806306f0cfc44515460c"}
{"family": "deserialization", "fold": "outer-4", "group": "cps-it/mailqueue", "label": 1, "pair": "07d1dc4d299a7700", "unit": "0e664e274dfb438d2f721edc81f8469ac48387dbd5c1b43973cd7e651db46db3"}
{"family": "deserialization", "fold": "outer-4", "group": "cps-it/mailqueue", "label": 0, "pair": "07d1dc4d299a7700", "unit": "da55f60378c67e91683b9720f2fa0e6659e28c7b392c050278b40713e47c84a3"}
{"family": "deserialization", "fold": "outer-0", "group": "datapizza-labs/datapizza-ai", "label": 1, "pair": "5a91049be21aa611", "unit": "1ab9936627aa59c3a60ea44074ad44b68aea4f9c04f14c1949dd9c77ca13ff0a"}
{"family": "deserialization", "fold": "outer-0", "group": "datapizza-labs/datapizza-ai", "label": 0, "pair": "5a91049be21aa611", "unit": "8df72b9eab3f2fd858c40312d0a806cb67c620bd62873826c8732ba8795d9cfa"}
{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "", "unit": "115d87f37bf1d4ab91fa3010303e81b127f84b817268b60b8fd4b9934efeae4f"}
{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "2b8101c89c839614", "unit": "7bb7d858a670fc51f81e08c141c2bca3b8b7524254ab679e33b22ce79536e895"}
{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "c23acf4f38945093", "unit": "82bdc0b6bb0c19e342efa63af0c9014a607461d0682cdd14bd5fea33516fbd50"}
{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 1, "pair": "2b8101c89c839614", "unit": "886285661b88f14e00ff8e863b31cf7b7afb303ad4aa4aa13b1cffbf1272d5e0"}
{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 1, "pair": "c23acf4f38945093", "unit": "bf74aae5a48156f183426a80cafca6da219ab13b2ae4b40fa88fca75acb72980"}
{"family": "access_control", "fold": "outer-2", "group": "delmaredigital/payload-puck", "label": 0, "pair": "d7cc37a3a54315cb", "unit": "4aecd43c6e5b81daf7aadd6a6e22f22c17c08b652a2f5121a49320ca1057b186"}
{"family": "access_control", "fold": "outer-2", "group": "delmaredigital/payload-puck", "label": 1, "pair": "d7cc37a3a54315cb", "unit": "88123ae860be2fdc28fd6a289cf26c3e0af0689f455cf8c15098e5dc76e5c8a8"}
{"family": "output_encoding", "fold": "outer-4", "group": "devbridge/jquery-autocomplete", "label": 0, "pair": "81caad96b221c2b8", "unit": "3c2dc720ed159e55629ba737665d93859def9790427289567a513c83c133e236"}
{"family": "output_encoding", "fold": "outer-4", "group": "devbridge/jquery-autocomplete", "label": 1, "pair": "81caad96b221c2b8", "unit": "e8b9595a320776c8e663fd0b168193c1e64e36a9052784d1219f2c4a91d647f2"}
{"family": "path", "fold": "outer-3", "group": "dgtlmoon/changedetection.io", "label": 0, "pair": "f9b6599018ef3b60", "unit": "7558d6ee39a311c5459e35fe6f7b12917b7f843b21ab1fbb5e485a07f53b7d78"}
{"family": "path", "fold": "outer-3", "group": "dgtlmoon/changedetection.io", "label": 1, "pair": "f9b6599018ef3b60", "unit": "b1a6cf450a074788ffca2bebd880435abac2c376e89cb328d7a22d277fdccad8"}
{"family": "output_encoding", "fold": "outer-0", "group": "dicebear/dicebear", "label": 1, "pair": "eaf2f03db2c502b5", "unit": "38dbfbf745d7cbcc52c6fff83248d84663273fe6a1f1b014eac7776c4bb8f074"}
{"family": "output_encoding", "fold": "outer-0", "group": "dicebear/dicebear", "label": 0, "pair": "eaf2f03db2c502b5", "unit": "a84d848ef56b72fc15872d88acaae1743c9cdf88095d13b1869abbec6a0d8a8f"}
{"family": "deserialization", "fold": "outer-1", "group": "docling-project/docling-core", "label": 0, "pair": "7df48bca2c9fabfc", "unit": "7b76ce8f55d61071adb367018cc3e50f04a05d08c7c4919c3806fc1cc8bf41a7"}
{"family": "deserialization", "fold": "outer-1", "group": "docling-project/docling-core", "label": 1, "pair": "7df48bca2c9fabfc", "unit": "c82ea0adc9c8366aa2b77ce73772038676fc37ff50879e873ff397958fdd7ec1"}
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "7f02a09b8d8f2321f2cdade74c2fa1d52442e006149a749a8be768daaa62195d"}
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "9b5e439943ce5c2b04d6201b7a843634aa1b13b726813c23450ff54ce03f3dd6"}
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "5b836889b4da4921", "unit": "a84c2ce4a4eadf265d4481ad7f6864edf1d7e3084c6ee32cbbfe454bb2946bfe"}
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 0, "pair": "5b836889b4da4921", "unit": "caeed7f097a3311fe2e84efd372fe4b4c140f63a80a9f07ba0b61966dbab15e0"}
{"family": "access_control", "fold": "outer-2", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "c2a2c2d85114a97f892fb1e5500fbd98622457a8c46451e1f9e958ab08a40318"}
{"family": "injection", "fold": "outer-2", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "29d8be8206c88ba7efd86b56a7dcf8b655e33b4a792c2d18b028c92c6252daab"}
{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 0, "pair": "f96f0fb245948915", "unit": "0954e6931373915c204b9ec8d2b8ae6297a95667bfd41aadf9035625ad93967f"}
{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 1, "pair": "f96f0fb245948915", "unit": "5e09cdae2504e834d591b6f92a808427f07550364b5419899f7e6f76084fc377"}
{"family": "output_encoding", "fold": "outer-4", "group": "ether/etherpad", "label": 0, "pair": "bf11f138fe39675e", "unit": "56f0d0b156992af56141d6368fdb0c5376fc7a44a03991d754f97202b9d77008"}
{"family": "output_encoding", "fold": "outer-4", "group": "ether/etherpad", "label": 1, "pair": "bf11f138fe39675e", "unit": "8271af5017894167556893da2070188060d06eebc87f27db45c46234c359631d"}
{"family": "path", "fold": "outer-2", "group": "fastify/fastify-static", "label": 0, "pair": "9d937c2dffb4aadc", "unit": "01ac12bb06e660d0aeef8b1e26dddc5c99d52cc333f7af6d1981eee2818e64fd"}
{"family": "path", "fold": "outer-2", "group": "fastify/fastify-static", "label": 1, "pair": "9d937c2dffb4aadc", "unit": "9612edf0f927048335e83e9925f01be9aea9b772bdbdc568ad9cc0a17055662a"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 1, "pair": "d4f648048e45f2ea", "unit": "08807c8110b8c1b8635a6883c229fb91f2b9566db8b6c252448a0db7b2ddd47e"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 0, "pair": "d4f648048e45f2ea", "unit": "10869c931053bf77ecaa98a47f56dea27d643d1abed2a4758bcf48a78547ccc9"}
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "662e1f36c230d7f1", "unit": "0b73332b25413902c26d5b5dee6767465eed99ff71a814582988ffe0d8ec1a30"}
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "8c8e46d0c7d361b784baff9d70f38744b2805ef1ee0aaacaa616bc20a2c6f32d"}
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "aca938682c196fd4c85b39b89eeedb97c35f6a5a017651fe9c03c5b727cab7ea"}
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "cec00d1509a129364da6ab8537fb3f433fd847e26d073c90311ccdaf7a1b39e7"}
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 1, "pair": "662e1f36c230d7f1", "unit": "e0f58a7ac5627de3e7b6cfe5add41d18b0f64b998c1cc788d42eb2cdbbc3fe41"}
{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 0, "pair": "2c251a645094f389", "unit": "0d41cc634771048f72d07c4aa9feaed669d0bfe3844d61dcdc525a0f899617e2"}
{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 1, "pair": "2c251a645094f389", "unit": "ca8288d0b8b94dc6b7e4c681936ea610d295fe949cba5f64f09b6de134ef9875"}
{"family": "access_control", "fold": "outer-2", "group": "forgekeep/nebula-mesh", "label": 1, "pair": "2ac5ec5011615c9b", "unit": "2f427f403af3cb58d776437736c2389805543413fdb91aad98979158c6d00de3"}
{"family": "access_control", "fold": "outer-2", "group": "forgekeep/nebula-mesh", "label": 0, "pair": "2ac5ec5011615c9b", "unit": "cddc8c5bed0841b80e7b20d0b378e858f8e4ac37fd7b890ce76611d989857f92"}
{"family": "injection", "fold": "outer-2", "group": "frangoteam/fuxa", "label": 1, "pair": "", "unit": "0c7830c2912ff285855d8a3c8e9b47db4ca4af840baa66d4b5d3f4422f46f95d"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "5383a8c2a78fabd3d17a5b3021125953b5ebbee61f8900c30d016c56b1a3bad6"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "360d8bfd2f116e98", "unit": "7c1989a4ec3622bdaebcf1161de850f22cfed041cb38830ca1ded4fa99d40466"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "ec8c706cc2b20f36", "unit": "848688dcd0fe8d46e6762cc8aee1d82e8fad8a2a044c94b547aea11049c85bfc"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "ec8c706cc2b20f36", "unit": "983cc9231f91baee4e2d626193f3b88aa5b09907f6faab870949bdc9c336b816"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "e4ad86416d5c6668", "unit": "98ff1dff2afe80709b8e61a9c95cfeea12b2110c13b7120478268b4762e2c992"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "360d8bfd2f116e98", "unit": "a1a825d23e34757c8e961e43ae791253e62682e130acdb846b58118588006a98"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "a7fa4207508b3989baabc7a1f0ff2ea5229b366c1cb106e7fc945134b8b106c8"}
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "e4ad86416d5c6668", "unit": "ef5c53f3114d0057e694df13dc07cbcb532c3b0d59150eda269b87b021d7a6e8"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "getgrav/grav", "label": 1, "pair": "9ae575048504c690", "unit": "176acf077fd58c3e6f940753179ffa3e8316661f355d94acb5e0345b51925ca4"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "getgrav/grav", "label": 0, "pair": "9ae575048504c690", "unit": "228b34ebc2090b5239c7821d28366ddc663e7118d015a79b45dca6618d5490fb"}
{"family": "output_encoding", "fold": "outer-0", "group": "gofiber/template", "label": 0, "pair": "a3fdf9ce3441ec86", "unit": "653236018929ab8534c27704a20ec5d358047bc66e451620d2ac32081c91d333"}
{"family": "output_encoding", "fold": "outer-0", "group": "gofiber/template", "label": 1, "pair": "a3fdf9ce3441ec86", "unit": "b70ca2c43f5929e7b885bd728349e721fd85d7605e2f722a7e8cfe0f0d477b0c"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "groundhoggwp/groundhogg", "label": 1, "pair": "", "unit": "92400e83890882a66ffdea1a405e2354a195442e176d85fb707b0442c44d45ca"}
{"family": "path", "fold": "outer-1", "group": "gtsteffaniak/filebrowser", "label": 0, "pair": "f9dcf3a0fb0f0a14", "unit": "7ca6dd4f673976b000a657f885af3463740419ce41e14f972af0c5483cb75a7d"}
{"family": "path", "fold": "outer-1", "group": "gtsteffaniak/filebrowser", "label": 1, "pair": "f9dcf3a0fb0f0a14", "unit": "86c1db3a5e9f843d6546146449171e0378ac61d6c8f44c6dd72fd379cdbb188a"}
{"family": "injection", "fold": "outer-4", "group": "handsontable/formula-parser", "label": 0, "pair": "6144baeb491f4a56", "unit": "f5c22ce104e9593e5272b6e8b3c6f3b61fa4578552ccea80e6021ee90bdb3ffc"}
{"family": "injection", "fold": "outer-4", "group": "handsontable/formula-parser", "label": 1, "pair": "6144baeb491f4a56", "unit": "f687e7e95dbe47f602a1e0002a44f94fd48e5b1c33f77ecebc6c2d814fcbf450"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "heartcombo/devise", "label": 0, "pair": "2ef4471c66e5e4ab", "unit": "ad7c8e368fe1699a044e058d5c10b41bd9fc0f91c3f3c384c5dac40af39e71f6"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "heartcombo/devise", "label": 1, "pair": "2ef4471c66e5e4ab", "unit": "d888baf7194bbe7158cb3932f58b47a9a235351d388d10df82a130ddd54c2d46"}
{"family": "output_encoding", "fold": "outer-1", "group": "honojs/hono", "label": 1, "pair": "5628c54152216646", "unit": "47d081c9f6060218967902f694e61c88f64f57c65bec6fc3073afeb0b77d117c"}
{"family": "output_encoding", "fold": "outer-1", "group": "honojs/hono", "label": 0, "pair": "5628c54152216646", "unit": "e1297bccf9c3606e0b41d762a69ff5d0a33ae4f7f1506071af6b9cb4a177ff56"}
{"family": "path", "fold": "outer-2", "group": "hs-web/hsweb-framework", "label": 1, "pair": "e52aa814c36ace1f", "unit": "2d6d74364f1982e91f27f1c995b8699df1fc8f8dce6e1e2cf274bf01b6e9494e"}
{"family": "path", "fold": "outer-2", "group": "hs-web/hsweb-framework", "label": 0, "pair": "e52aa814c36ace1f", "unit": "4727471fea2cf0a92d43b4b1503f1339a7dde87f872ba8064dacbf855c57705b"}
{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 0, "pair": "6548e1527e82bd99", "unit": "5542613f6d2eb9421b8213b8bb864894f0ecf9fe64945681019230ef0d75cfaa"}
{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 0, "pair": "", "unit": "6c2adee34e9e8384535ec1073eda60e2ab528ac77c36747802c303e1f5e9ffe8"}
{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "7ce70e9b20107cc7ebb87d173da765fc53ffd268e521749a06a9a1f38d746845"}
{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "6548e1527e82bd99", "unit": "c3e54975464323e58ec96ee4d83e7bc08baf94553d553a8dc3fb28cd9c70ff21"}
{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "d3173c1e43eb6402a71ea2dec1592913dc078a2194ba1c1d509ab129a3180f6f"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "internlm/lmdeploy", "label": 1, "pair": "0924befc49c7bb18", "unit": "e7e4872db40f0420547ed622fbdd7a072b64a7ba466d1ccfff68e691cee6ad99"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "internlm/lmdeploy", "label": 0, "pair": "0924befc49c7bb18", "unit": "f7d95f43fb2df20ebc4c37fceabdd00ffd8159f404ca384481f1764252382388"}
{"family": "path", "fold": "outer-1", "group": "jarofghosts/glance", "label": 1, "pair": "772e18821701d60f", "unit": "89a0496c7663bbfd67d660c9f481c12420da2dd1bdb29dc079735ee7a0a49626"}
{"family": "path", "fold": "outer-1", "group": "jarofghosts/glance", "label": 0, "pair": "772e18821701d60f", "unit": "d3241ceaa50a7d23a4650e6a2a55372b08ec96b35b0bede844bc40e16e818ebc"}
{"family": "injection", "fold": "outer-0", "group": "jas-/node-libnmap", "label": 1, "pair": "250de1302c7e541d", "unit": "5d03535f2df2acdbc319d9e6b20da65d51f3e810b80f3c8f89b5074a971a3f1c"}
{"family": "injection", "fold": "outer-0", "group": "jas-/node-libnmap", "label": 0, "pair": "250de1302c7e541d", "unit": "78f977cfc096e34b4eada7f798ad2973ef45728d0d01bfab6ac9e2a3257827ef"}
{"family": "injection", "fold": "outer-0", "group": "jashkenas/underscore", "label": 1, "pair": "c10bfb006f2da55a", "unit": "1ea4b86dbd84581b1d7b7cabbab9f28c8b213416a60778879666ff42d23fd7f1"}
{"family": "injection", "fold": "outer-0", "group": "jashkenas/underscore", "label": 0, "pair": "c10bfb006f2da55a", "unit": "7647021f742da24c660b2d407886771786e957076c4cf336def560a570dd8778"}
{"family": "access_control", "fold": "outer-0", "group": "jellyfin/jellyfin", "label": 1, "pair": "a73421efb106ab0a", "unit": "1f6f207a758582de0c0e6b331b19fe119abe18727d45ccbbe09aabb795f3a292"}
{"family": "access_control", "fold": "outer-0", "group": "jellyfin/jellyfin", "label": 0, "pair": "a73421efb106ab0a", "unit": "32fbdad39eaf5177db4ad0a11d53e785b30bc7b2376ad9d460f29a08429bd0c7"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "jenkinsci/cas-plugin", "label": 0, "pair": "0458e3fd444409f9", "unit": "b521216bb4106738b3571ad3902d08f6230aabd88cf3a8f9f26f1c8ec01304cb"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "jenkinsci/cas-plugin", "label": 1, "pair": "0458e3fd444409f9", "unit": "c87d2871907798fa2e73d31381e9d63089b0388033f096d247373fec07c2cb36"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "jenkinsci/google-login-plugin", "label": 0, "pair": "6bdccabcf5b9fdc9", "unit": "3d6ddca5eb264d88411b86755909f1d2abddd09c129cabd54c96ff26b4105316"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "jenkinsci/google-login-plugin", "label": 1, "pair": "6bdccabcf5b9fdc9", "unit": "e055b898802279bc63204cc601e29f7f1018efad7228cab0cce00ea5866ecc7b"}
{"family": "access_control", "fold": "outer-2", "group": "jenkinsci/mcp-server-plugin", "label": 1, "pair": "91bc832e70680c92", "unit": "1060c63ff0fae3af567448f3d4c60d9c159b61ccc358964243eb152a9ac38865"}
{"family": "access_control", "fold": "outer-2", "group": "jenkinsci/mcp-server-plugin", "label": 0, "pair": "91bc832e70680c92", "unit": "369dffd1e730d7997813adfbebf1481cc68987cae0d0085e47b91a26d5330cf8"}
{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 1, "pair": "a4144258dffb3b8d", "unit": "7ab301e1614979f44c353e665540582493f78ef4bfc41209562c19b900710f8c"}
{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 0, "pair": "a4144258dffb3b8d", "unit": "ea95e8387dff32ded9dc7b04c11a76a545b4e2ea6db0076b9b13971a59904bb3"}
{"family": "deserialization", "fold": "outer-2", "group": "jmespath/jmespath.rb", "label": 1, "pair": "858330e01d3cf9bb", "unit": "033cda2b7cffa5a5fa6abf0b7f2cd27ba78d916eb3cab8c79decccd8776f4a3f"}
{"family": "deserialization", "fold": "outer-2", "group": "jmespath/jmespath.rb", "label": 0, "pair": "858330e01d3cf9bb", "unit": "d44c4555a5ae19aa17b468ec5cff56b33a0b221df52e04d8abcb656399499dee"}
{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 1, "pair": "c6a49339d4f0c798", "unit": "dc2e4cd5bd963643702d2b832ae170dd25744162e4076e36687256a09e69d089"}
{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 0, "pair": "c6a49339d4f0c798", "unit": "f83cd137b85e651ca5299c5a42176b797ed4964cbef31418c96d63c52fdb65f5"}
{"family": "access_control", "fold": "outer-1", "group": "jsonbored/gittensory", "label": 1, "pair": "5ec37c0f475001ca", "unit": "5d8175fb21a4649bdc8730358e9b87adb90b2aafc4da5fee203829663e4f46a0"}
{"family": "access_control", "fold": "outer-1", "group": "jsonbored/gittensory", "label": 0, "pair": "5ec37c0f475001ca", "unit": "af845e9e332555096dd350401977b2f07015433f3db8a040891d9470f3f9e355"}
{"family": "deserialization", "fold": "outer-0", "group": "kevin1024/vcrpy", "label": 1, "pair": "c74525e844650de2", "unit": "2e444ad34ee5f47d73f0cfad0a10c70363337b4ddf9067ecd1b8072d8232652c"}
{"family": "deserialization", "fold": "outer-0", "group": "kevin1024/vcrpy", "label": 0, "pair": "c74525e844650de2", "unit": "c492d325f5ff82b5f817aa4be1440cec8595054dd0719e07a58ac5501b2ae7eb"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "koajs/koa", "label": 1, "pair": "111a5a80a3abc2c9", "unit": "c7ae403742450deb2bd051ad1231121d2117ba56fa72a038c82d58d207cb3870"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "koajs/koa", "label": 0, "pair": "111a5a80a3abc2c9", "unit": "debe34831afca6f8220b9952036a2d4570babdb5e321a2f8dbaec661a016bd50"}
{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "9950b85412b021ebabb11eea4c490a660ad380d14c673a87c3fb568ba4bcfac9"}
{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "656df5ff8e69dcdb1f9ba1ed52d734966eb29773bc745587483a355743b8eb5a"}
{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "91652a4e776fbce70cde9e4a99c6da5e17da7f8d6d0404867c61eec04161b6cf"}
{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e53bb3a217dc4737e8dd03671de4f6a895599024070a985a268c65a034295e7c"}
{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "3d7aba22444811a337abcad0e4fc62990bb704d3a10e72d893342fa94699026b"}
{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "5f18a3026a51f7140b7aaa3aa8563e8b6fdda6065d95e19e25b1dac4b9ce0e00"}
{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "0fccc306ce80cb4cbfdcde0a2e56d637bcefa9dd30423d6c54a4f77860afe740"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "1e2eaa45661345f1b6759d968d7696dd0d96d6fba1350a00e4d0a2ebaaa2e391"}
{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "8b9a8ff0e32a327df75eb8a1811464d5f34690079d5930139dd917063bdd2809"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "245d0e939b7c1ccf098c798492009e3a73ea17400fa613dd865cd369d2e7f383"}
{"family": "access_control", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "653c48df493f31c2d01c973d1e379a180fbe6a6edc83a500bc62509112175f41"}
{"family": "path", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "3719fe21d45b50b35458bcd2e263e4baee67adbd3c98adf0eaab6bc6183c5651"}
{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "1209a6cc8c1548ed83d4a9a09d6a27f706a23e69cf0c261fc5e3a27536a83306"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "362c90495dc468bb732c71f62d71ec308854161de31ad18d2a870056d947ddb6"}
{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "b52e53fec9707b2b442c4dc7a41d376cf76946d2c45c2eb600c9477fc6baebaa"}
{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "477e3b90b92e622888dbe0e9d3a23b31271ddfa3104a5ab326eb9e9e765aa01d"}
{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "660c63c4a73bfe64088ae3249e40d9c5d69a6f8a505ff671217c5708a345c5b0"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "4d60c386c801bedb61c1461fced29a3cfb50b09d6435e8617c60398343719d93"}
{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "f1fddcc1354a29bef55610ee2b73eaa440192f2f18a0d22d08057b9d1e1b61b8"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "6d9886c38221b2f9aacf83749218ac5ebc354a6fcc4170e6c78bedb92b88296a"}
{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "5550fec4a81c4c9cb22d7312d8a63e1fb2b03c4aed0c1f316d44c1eb27511bac"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8ff87dd04140f0137ea1891f220b1276156ce10a58be5a49d0c8e07be531ffb2"}
{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "c6c9bde6630678377d250701073c5c223c7892e6ff8f2f8e0293dea22c5b5e7e"}
{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "b9b15958973c83f8d7b58372ab5ba58eb191c400125579d5ae8c5b52d5d1368e"}
{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "b95ec12e4e126ad558e944e2c260116ed6a63f5f3ab97fb5cbcdca75a931ad73"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "38a5545ac9ef67e15db720d1396a2d5915f9d06d6f4772012e5bc95f33770376"}
{"family": "access_control", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "c8bbe9be91c976be97b7ca387048a7d4537d9f6fcc6cbad0a541d1080629d5a6"}
{"family": "path", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "ffacd53f1bd75155e63b318fb9cba27ec03e754841e1eb39d1d4fa0b2c3dfca2"}
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8fd920175fc43b91b4397acffacb10ab8f7fd0ee38bcf2d2f3ed06a3fad0dfe9"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "86c9598ebe3dcda261a69ae545513f6ce73101f3980066e4506ece99bebea84f"}
{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e5190fcc57505116f79a31c51aed69f8a1eea13a8ae3e7265f44c72418277ef9"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "2b807ff54e6c47e941124c16d0c27c691cef54f2203060a3a6ce812d478c5261"}
{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "401aefb4b4791301ace2d770d321c6b26743b0aab7b96f8b410254c704be6d2e"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "22d186c93498e21418b6c1d0547a73374b13041d8afc89a80d9540fa658b6b42"}
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "dbfb6df14fe37f8c0efda5f16643297fb0550dfc5ad5805d4e4b844d967773b2"}
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "f0da8cfe4ac1e586aa7903d72e67b44500bbd5306d72b4b342d10613efc69222"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "2b8fd73943f456da266434470333a7775f55d9a41953e97a000fdaa3257b6a30"}
{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "a612091f452e60f28f44eefec641175170dc5db04456a39a099cbb13c50b4d2c"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "4c6600facd83bbf234a32d9e28ac494fb516df6e3d9e9dc28d720ff35252d067"}
{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "fb0ea8cab8c9430d22d1b0fdd65a69146fa95fbff94e720b4861b96538d992c6"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "e81f91a56058ac1473579767abea57064c53d7d007797a854314814e3c35ee74"}
{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "5851ecd96a449f98b05f243bfda0a7cf3af8612b27f53c414ae1f4ea67242a7f"}
{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "328621ef2a94632e5d37ea6267d17b766233384cde35c07e29a4eb672d40ba28"}
{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "04b2eae2128608dad460ba8275142a41a44e32c4243a29e2a6fb2c9ebf3d8f1a"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "33337002ff5f5a189610b042522ceea254c9d107c1f24771b4f616a0f96f636e"}
{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "02c09a783cbf058856fc2b448406f3ffa6fbfbacf9b70f4e5e22fd741a5c7699"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "956260164cb21a47c499e5598491031ef303418943f5096f225a87162b588574"}
{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "8bd8c5aed69eae3019cc2d3654b36bdf2b31435e6a020a389cd103696a2f874c"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "1d8397ef4a9fde7b48e2e1e871bba0f960a802cd7823ecfc10e9dfaddb955ccc"}
{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "c9db92476032eb4296b950a1eff8fa4aaa26f11542a01a5cf10c7b2e32dc6433"}
{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "d3e0d0f9b0f4b723080989b5486395cda0a5b186a674022ea70e7fa7ab561323"}
{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "44ff3b3e61c02744830c4abea872821ca7e0f94950c082740c5c48ef13604e94"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "4103e979a99233c3e71fdfe9f1cdcbab34a2c60b5351b75c5ff5dfc287b962d0"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "kozea/weasyprint", "label": 1, "pair": "6768bfae38c6859d", "unit": "f376519889a00950f5af67eb15c639306ea70da7235d3be4a94dbc19cb096cf5"}
{"family": "untrusted_destination", "fold": "outer-0", "group": "kozea/weasyprint", "label": 0, "pair": "6768bfae38c6859d", "unit": "f3fea4e334f49ef6fe6b17121fef553eb8e33a9ffa13e6d6cbc4c448da996a7c"}
{"family": "access_control", "fold": "outer-1", "group": "kubev2v/migration-planner", "label": 1, "pair": "d670253fa209671c", "unit": "1aeebe2e440bdd70525c31f93180f33be93f820db5ba46a2e59c371ef83ee794"}
{"family": "access_control", "fold": "outer-1", "group": "kubev2v/migration-planner", "label": 0, "pair": "d670253fa209671c", "unit": "b11c8a8a24d3809c0507228568fe7cc05ca0a22ed28cb19019adb13cb619d63f"}
{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 0, "pair": "33260f3e6c8ea6d0", "unit": "7ca409c7dbc1c38ff005624e212efc6d424cb421e27b482dc8dc0d6881f4c11a"}
{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 1, "pair": "33260f3e6c8ea6d0", "unit": "fc364d7ed77c7c0508da00e5ba847ec0792f3bc373d8b4297747189ef42d1ae0"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "", "unit": "6cce78d7569ad8db7ca0a61d7078198f666502d26ea6c0afb03173a8a4bb16ef"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "82047c0513e43a83", "unit": "6d2dbed1ab1c9d8ac259dd4227be69641fa59d52e816a8dc547ade01d0b9b0c4"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "98ee23cd6bd31cd3", "unit": "ab4bb39be535d70cd95e9035bb958bfc7418576a6ff40a47551894dd140317bb"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "98ee23cd6bd31cd3", "unit": "b70068cbc646bd01d3e10986e002f648a74e9d798900dc59cf3039b749c95c18"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "ff168d95d8d9496e", "unit": "c6ec9478c125ed780d7013c2807f2cec1878f44253667dfed319c28cbfd17d4d"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "b1bfef46dcd3b9a5", "unit": "cd4d095d8380a2fb7ca02fc6abdbae5249cc06455db2929875533f4aa8b886dc"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "82047c0513e43a83", "unit": "d1bade217be8d345cf5d0f0442ee31924187acbb98779dada95274a77462f696"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "ff168d95d8d9496e", "unit": "d561db250b892b91feed7ae877900e1e25963444f73c7699576e010f06da73cf"}
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "b1bfef46dcd3b9a5", "unit": "ed78fd5a7ab30bcd6e8ddd27c706a19cfd89750de4fbf81c0b567946eb94db83"}
{"family": "output_encoding", "fold": "outer-2", "group": "lepture/mistune", "label": 0, "pair": "25f75d77a669b54e", "unit": "0f2825568461562f2a915a4f16ef8a4931290fd20a9801470aeb12677e66e3f1"}
{"family": "output_encoding", "fold": "outer-2", "group": "lepture/mistune", "label": 1, "pair": "25f75d77a669b54e", "unit": "bf22bb44bcff4a5baf5d550c9d3e83e5f3ca607475fb51deaf4fb6fccff3d73a"}
{"family": "output_encoding", "fold": "outer-4", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "cd80750447cf7af67ad809279718d98106c71705fef9ff7254728bfff4ece5b1"}
{"family": "path", "fold": "outer-4", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "2ce605ab5ce7ad3a1fb983cd6bddf3924e5da3010d2252a0a4640aab4e5921a0"}
{"family": "deserialization", "fold": "outer-1", "group": "litedb-org/litedb", "label": 1, "pair": "b0a899116dc36b85", "unit": "510ad48af03ba4a1a2a06dd254b135311dddb62d9fe5a2e8e8b12627586e6b13"}
{"family": "deserialization", "fold": "outer-1", "group": "litedb-org/litedb", "label": 0, "pair": "b0a899116dc36b85", "unit": "b414718a8a5f79739b88db24da6ad9da8e48bcb6b5e2dc99158848cdaa226767"}
{"family": "deserialization", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "8e22ee30441b20484090659b0c97423c711f5d10219041bf3a399623fc056a07"}
{"family": "injection", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "0f9070df9dc334f981cb01008c260b8bff78dbdeb48ce672830ed95caa8406fb"}
{"family": "injection", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "5e352f797b1422b0df477bce4730a9ce33cbd0837c3d63da94e4d8801a9a4b64"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 1, "pair": "dadccb128ac0e120", "unit": "505097494da5939404ad7d5b4af2b1c4a10efe94048084492d6ce60ebaa04c97"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "dadccb128ac0e120", "unit": "84ff8ca2d98a1b5d1ca4bfaabe7d19e03a396042b228c8d18d1185a178389702"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "91e261c54bd2c3c3a9ac04e48894ecabaca5b13e7db4fbd5578f9d266098117a"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "a308dfd59bc398824e2454bf6cf6cfd5b7880e5b948a82424f14753d2b6fcdbe"}
{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "ba24eb554458bc97314ed561a7b2f4a4e643dde1205c32de4c8fcdd8c46e575b"}
{"family": "output_encoding", "fold": "outer-0", "group": "mganss/htmlsanitizer", "label": 0, "pair": "24d8f1840b67fbe5", "unit": "7ffa4a229a62266788fbf64890f81f20358b3eb8c35d3ab87d0c339d5bb4b507"}
{"family": "output_encoding", "fold": "outer-0", "group": "mganss/htmlsanitizer", "label": 1, "pair": "24d8f1840b67fbe5", "unit": "d19493ed7941e1cd25a76dc50504d97398770d07ebc302dd66d83441e946c468"}
{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 0, "pair": "9f8a60eb48d74bc8", "unit": "80fce34f2377731e3de4355c7ea33bb20fc2fc00a5f388565b23018c7a3ed008"}
{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 1, "pair": "9f8a60eb48d74bc8", "unit": "b68e62722a327c9d5ce6774eb96527451729410768efc0b5c7ba604ec468052d"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "miniflux/v2", "label": 0, "pair": "997f9d2efc32302d", "unit": "b587bb499945a9f5f0749eb4d57cc1027cfd21c8292bd32be4d4e1cc1b1db261"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "miniflux/v2", "label": 1, "pair": "997f9d2efc32302d", "unit": "f8d9c9a7c12797b3caf042e6bfdb89a5072c4eae1fd36be6196f9fa326cdf257"}
{"family": "output_encoding", "fold": "outer-4", "group": "mixxorz/slippers", "label": 0, "pair": "8e6b4b4e28981590", "unit": "3cce3e8f9ff3f1d261f9b96c844c1e4da07bae72d5393897b3a929be25372edc"}
{"family": "output_encoding", "fold": "outer-4", "group": "mixxorz/slippers", "label": 1, "pair": "8e6b4b4e28981590", "unit": "b105689ac24ed334294c434a77bb6d56d5d598830ffe8a5d60ab3be5290b47ad"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "0a36cd5177f3b2c20b433db6bfffe5ec5b7e74cacc34a86ac581f7a0a559432a"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "e7fc97d83013c9ff", "unit": "5ab444e64297fc19065478104c7ebdabe23cc415f22adafd0cf82c98e4c5cae3"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 0, "pair": "e7fc97d83013c9ff", "unit": "6b462ea30c1afda572600a10324249223932d36a032eacafac8c837bccbdce87"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "84d6a4202a082e0d", "unit": "8175082f40d8d61d44b54579c84a808e25b89110be5d8fd82feedf7dd5e0d5ab"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "8abefca43ee4ae2c797033da31f85c17bfa7be6b35413d70b0216415c5db9a09"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "91d213946859f33d3f243c43aa8285cc6e8ecb3db1de938f32af775a70607f8e"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "a244d9f010ddb43a09e82363dfabb0e406d411fb5de08ab7d6c9ee367f2a15c3"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "d27d174d6e8e5f1682a1c42723705e89f0d18c95ecedd0567e313daa9a0aae44"}
{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 0, "pair": "84d6a4202a082e0d", "unit": "f45608ed47fa11ca25093641f6021ec06076c18676abac9b809f31b3b7713de8"}
{"family": "access_control", "fold": "outer-4", "group": "moquette-io/moquette", "label": 0, "pair": "55d1ce4f29dd1583", "unit": "a877ce3462da316bcdd82ce9db1d9887e887eaa83157922edb1663413edae1d6"}
{"family": "access_control", "fold": "outer-4", "group": "moquette-io/moquette", "label": 1, "pair": "55d1ce4f29dd1583", "unit": "aee67be0ba31f99c7a24faa1bad6178c6d37361a308cf5aa10138e6ce4bbcfeb"}
{"family": "path", "fold": "outer-0", "group": "naorsabag/openhop", "label": 1, "pair": "16e529fe8c5486b2", "unit": "1d21acd0171726775d3da4589b660645fcd43cc026042a2b27e0e0e8374eceb7"}
{"family": "path", "fold": "outer-0", "group": "naorsabag/openhop", "label": 0, "pair": "16e529fe8c5486b2", "unit": "f8e004b67ac98dc3392ea4115051e6bd10ce22eb638a762f0f16e1d4200bf129"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "nuxt-modules/og-image", "label": 1, "pair": "ef603436079b0005", "unit": "77f843b5044bb5da935df4b2620bf2abe9e7dceb3147439e883927112eaec22f"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "nuxt-modules/og-image", "label": 0, "pair": "ef603436079b0005", "unit": "f2ce95f0e44e73986784b63a702ffa4035c423056cac3667a63140a7e8d795c2"}
{"family": "deserialization", "fold": "outer-3", "group": "opensearch-project/opensearch-ruby", "label": 1, "pair": "1945f28a3fc5bd55", "unit": "64c0b7ccad09192eeaf590befe0de0087d5b48a65a4f6cef41415f3f54a8f536"}
{"family": "deserialization", "fold": "outer-3", "group": "opensearch-project/opensearch-ruby", "label": 0, "pair": "1945f28a3fc5bd55", "unit": "b55cd6759d4b7861e589052aa534bd0554586db46a329ed3beed7244d579e8c4"}
{"family": "injection", "fold": "outer-3", "group": "openssl/openssl", "label": 0, "pair": "1c9e47835232f644", "unit": "90e220d3540b88dd9efc8e66c8ab519d4cc78805f16bed585b795c78cfbb4d58"}
{"family": "injection", "fold": "outer-3", "group": "openssl/openssl", "label": 1, "pair": "1c9e47835232f644", "unit": "b5a185b05235f70f6cf27e2adba919f4d8393dbdf2d99de095548a21946aeaf3"}
{"family": "access_control", "fold": "outer-3", "group": "openwisp/openwisp-ipam", "label": 1, "pair": "9b2b34cf62a76ea3", "unit": "91a3984e47f17512db8523e04f863525c67db18631d8f11776cf61ca7221c91b"}
{"family": "access_control", "fold": "outer-3", "group": "openwisp/openwisp-ipam", "label": 0, "pair": "9b2b34cf62a76ea3", "unit": "d91f1e14282a0c3c8da57c6935a1625e881374bd205b5baa9e4a341d9109ae21"}
{"family": "access_control", "fold": "outer-3", "group": "oqtane/oqtane.framework", "label": 0, "pair": "a20cdf9dd208720d", "unit": "18a26df68e4b81d411cfbcc8155a7e6f382753d6e1ad95eece7ed24fa8c44324"}
{"family": "access_control", "fold": "outer-3", "group": "oqtane/oqtane.framework", "label": 1, "pair": "a20cdf9dd208720d", "unit": "e6fb74441fb585d7ef5855317fba795ef0f0a0e6eae9970a49556ce3070c3254"}
{"family": "path", "fold": "outer-4", "group": "owen2345/camaleon-cms", "label": 1, "pair": "d6930fe7e752b7ec", "unit": "06f0341dfbf1aa9504083737df5d9be3cedfd4e3a198799d8075a29990776800"}
{"family": "path", "fold": "outer-4", "group": "owen2345/camaleon-cms", "label": 0, "pair": "d6930fe7e752b7ec", "unit": "30e2cd8bd381d7568b78eaf982f562d278a6a8d942d5282466532c1441bdf0ad"}
{"family": "access_control", "fold": "outer-4", "group": "perses/perses", "label": 0, "pair": "2d482951d7380c4f", "unit": "5c1a721da6f85d1d5dfc8cbe5e9809b093a09c059e5abeebed2cbbee55fbd498"}
{"family": "access_control", "fold": "outer-4", "group": "perses/perses", "label": 1, "pair": "2d482951d7380c4f", "unit": "6cabd1c4b5aa94cafdd07c0c6a3f9abfdc03761ddc5b0a2bebc72a508d020b8b"}
{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "", "unit": "2300c097374b9098671dcbcd42d02d08172e0f3064a8323223d0fe79a6106aab"}
{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 1, "pair": "0664148b0b192456", "unit": "d702617aaf4d19811baaa18386a3c8b0eb35cd3fd72e8d90611d34850e990858"}
{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "0664148b0b192456", "unit": "f5ed13d811c5b16d46cbd4a3b9edb8f7d2db49acbd49d95f39f33eaab38f58f5"}
{"family": "access_control", "fold": "outer-1", "group": "publify/publify", "label": 0, "pair": "1972ea9f72b178b6", "unit": "3ddabb356d78e7c65ec8f88b6da8e4fcabc055ae73f2f6d92c50efddc5d4fcbe"}
{"family": "access_control", "fold": "outer-1", "group": "publify/publify", "label": 1, "pair": "1972ea9f72b178b6", "unit": "6875dd5e9c0ff8a1b693d29a109cb2e62e6ff4081df40868aca70e4770476757"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "04aa777130c02a06", "unit": "16aa43c87956c44a4b115245140eefed704095d37cec0cc6410377403cd2fa7a"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "23744137e0485a36f686f5f1b5666780fbed3ca522e8c6d35687a391ef504498"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "67565d99fe28c68f", "unit": "2692316792e6f02fb51578cf30d71bba6c73bf5420c36f709681c8eaa69807d4"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "2e36f73002c416a68d077bffb0335d82346c7507f2478353aeac005fce5a9cbc"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "04aa777130c02a06", "unit": "67721f67055eb6fed955ff22bfe4060344662eccb369431835a2116ad120c4f8"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "8fc1c96fa6faa4d3966ecc42876189b18ddab779d58f47cdf28a1155f48a6ac4"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "67565d99fe28c68f", "unit": "ab535098a8e709b022afee1da15dd1fc371d389056938ee4fa4971b2966c823a"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "d46f94235bc462360097e7a22e1d442d95f049306631489a4d8b25f647a95bbe"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "edd6dd4364442f66384d14e9317ef2f6234e32ab78327a45741b8a816b4332ee"}
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "fc7df6887ac719d4615934c0e8c3296728c2f6089f233158c59989027c802d72"}
{"family": "path", "fold": "outer-2", "group": "rack/rack", "label": 1, "pair": "c0412509f15fe3e1", "unit": "63bda2a71ad7155ad2f59aa9086efef3fd3710dc82c57f4b32ec8bf561460239"}
{"family": "path", "fold": "outer-2", "group": "rack/rack", "label": 0, "pair": "c0412509f15fe3e1", "unit": "b1c9a070c93764655786c3de5fde536c150e319efe47548614cc544b73a8ee2b"}
{"family": "deserialization", "fold": "outer-3", "group": "rails/kredis", "label": 1, "pair": "feed5f7de11289a4", "unit": "71531567b2b8609ae0d31cb177b8faea4f5fc61923e513b0154a2c941287ce2b"}
{"family": "deserialization", "fold": "outer-3", "group": "rails/kredis", "label": 0, "pair": "feed5f7de11289a4", "unit": "d6be6d397a301a07faa761163d3d9e270d71d23d3bdfd084eeafb1e62b91dd71"}
{"family": "deserialization", "fold": "outer-2", "group": "redisson/redisson", "label": 1, "pair": "ffd4b04a7bebfaf8", "unit": "54f7519d3d59b6d8d69a5f8f7bd78e360d3fabaae00b7587c37e388719169370"}
{"family": "deserialization", "fold": "outer-2", "group": "redisson/redisson", "label": 0, "pair": "ffd4b04a7bebfaf8", "unit": "c5e42c9ab2ee54e46700ab0b64e239b567fbc75ccaceaf6cbf20687813663332"}
{"family": "deserialization", "fold": "outer-0", "group": "remix-run/react-router", "label": 1, "pair": "73f2a014c6a699ed", "unit": "44ba9d9d8e75769c1444ebf37a64941a845b4bcae678ec79d64dc84e5cc067a0"}
{"family": "deserialization", "fold": "outer-0", "group": "remix-run/react-router", "label": 0, "pair": "73f2a014c6a699ed", "unit": "b430d488c486cef26b1c661f773e7e9e4a177eeea73537a033eec98d532e4eed"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "remram44/taguette", "label": 1, "pair": "cb0c638282db0d37", "unit": "a49e6ad2e7c2a5ad0d68d5d69c263306b501fc0dc9382e701fb5fe680611196b"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "remram44/taguette", "label": 0, "pair": "cb0c638282db0d37", "unit": "dee28099d3b37cbd0ee14412122ecc5f8488ed6bec01863520b87706a040fba1"}
{"family": "injection", "fold": "outer-4", "group": "roest01/node-pdf-image", "label": 0, "pair": "09bce271e8530fbc", "unit": "35eb19b4c6176b37287e2123bb98ec4045ce6bde7cc3864dde317c024b0935d8"}
{"family": "injection", "fold": "outer-4", "group": "roest01/node-pdf-image", "label": 1, "pair": "09bce271e8530fbc", "unit": "dd33291cc494bbc77d44e9b13d16f171d32c9746637506f315f375e8a9ddee95"}
{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 1, "pair": "40d8d81a764c508f", "unit": "7deeed73c29c74ccbed29be049e109d363d41e4b5eea245ee57780b85f0e137a"}
{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 0, "pair": "40d8d81a764c508f", "unit": "b5cdadf848e67910550666ba2a6ac67b24112e501c02dba1865766a316317a23"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "", "unit": "15dbea384b3255457a44d9382efc6944dd27f1d411d903344a5d5e5fd194b72f"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "b2ac284bd70dd684", "unit": "225890ba6694dc8f4fbab46e9c851dff8c095e906cf14db15d01ebce0891aaab"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "b6eca35de93a5be8", "unit": "42eb8427415704e060e73b5ce0943b014ddf380ca756d5ec47c558e59a79895a"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "b6eca35de93a5be8", "unit": "4c9cdcc11a18641ef41e9308712c5e8b830ec9ad5ebc3a956c8701e4f77b39c0"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "6a53939838afbdd7", "unit": "5794809e595311c86909d673a570d003aa025e4dcd6abc5641b641d935beb0b1"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "46d3ec7d49cb8ca7", "unit": "5f7a63ec650915962ee5255f5416df63ab882d6107f3b540be36d481798a8944"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "6a53939838afbdd7", "unit": "97a7018535ab511622e3ade9e53d570ffb1c80633eaa372fb76e749b3a586c8f"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "46d3ec7d49cb8ca7", "unit": "a84e9ab47f4f2b584684dbb81575758d9032fe3f2ed53bef478cb9dcfedaf677"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "b2ac284bd70dd684", "unit": "aeceb1ebc863ebbc3b69ae56c5b78d2dba005132b75cbd6af929bb0bfe25da9b"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "f42cab375ace1739", "unit": "e329d4a6ac757680bf3d292a81f1f74f8a8a9b482d1651efe9f503f6a4de8796"}
{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "f42cab375ace1739", "unit": "e73c5aac41c0b93072223a3097fb175548ce893e0678ff3a22bc7571745a17e0"}
{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 1, "pair": "", "unit": "52354ccdbc16f209e4692d97d5237e04ed030cf4a5e71f77b5a767962b355b4a"}
{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 1, "pair": "7d23be667536afff", "unit": "b5016a5a53409491ba805ed8718599b84250b7b6914da0434b0882b0b3578dc8"}
{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 0, "pair": "7d23be667536afff", "unit": "f1b01102968568d3ba1b2fdc70f4ae281f433fb20d29eff2a55fc45555ddbfcd"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 0, "pair": "1199de75f426940b", "unit": "10fdc236f43a02b7e291eb8bf6ada297a2ae98086a63b171057bc7bfc37a3377"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 0, "pair": "22112c31603bbe9f", "unit": "5eb3b7523b5a196dcaacba08d68457f9a45e31503ea02745ca3a7d0542f9881c"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 1, "pair": "22112c31603bbe9f", "unit": "6bd5bdfd81b0a3374d59cfb2c1f7fa2b27887f4672f4485e77eb5234b569983f"}
{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 1, "pair": "1199de75f426940b", "unit": "c09dffd04708540845c8d1dd77a2bd989bd35f264abf652a2db55f8cb1faf7a9"}
{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 0, "pair": "7aab025ea7b97019", "unit": "7487a34cdfdb532a5096764a47a3e565fa509605c1df0e2b105eb9c751fc5307"}
{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 1, "pair": "7aab025ea7b97019", "unit": "9c0be2a58741f095fd0a9cdc60d4ca4039de1529699d5701dc4d0bcf7d5bf90c"}
{"family": "path", "fold": "outer-0", "group": "simonh1000/angular-http-server", "label": 0, "pair": "4de5441f1776f083", "unit": "a3a35d573da308cd1d0614a26616bbd37b360d9ea416894cdeffdb59abd691d4"}
{"family": "path", "fold": "outer-0", "group": "simonh1000/angular-http-server", "label": 1, "pair": "4de5441f1776f083", "unit": "dc33fc26c1bf7c67e5439ddfdad06c0faf16c796e0fe8a46b6e6a541e708676d"}
{"family": "access_control", "fold": "outer-3", "group": "smithyhq/sqladmin", "label": 1, "pair": "d76d5232a8587127", "unit": "1ec32d427822d1ddd3b0d7245f5771ddeb56c146ac8ff9959dd85760ff659a20"}
{"family": "access_control", "fold": "outer-3", "group": "smithyhq/sqladmin", "label": 0, "pair": "d76d5232a8587127", "unit": "5143b19a1a1e6b9162b80594ed7fb5c0c57ed0b8abfd793bfc71089552d57638"}
{"family": "injection", "fold": "outer-1", "group": "soyuka/pidusage", "label": 0, "pair": "5f6e814444b4acd8", "unit": "2e8aa07b1aa4bfc5349b0d4464f3f2a6500aa55ff765b0139e9901ed34151ffa"}
{"family": "injection", "fold": "outer-1", "group": "soyuka/pidusage", "label": 1, "pair": "5f6e814444b4acd8", "unit": "e26c726c08be761b22fcc6ec17fc1fa7aba4769f08957f024e5545906df62797"}
{"family": "output_encoding", "fold": "outer-2", "group": "spatie/schema-org", "label": 1, "pair": "888e955fb6cfb4fd", "unit": "6f1a431ca97afc34ceec48393eb134a8928a312e6f761eb907c6b5855cac6208"}
{"family": "output_encoding", "fold": "outer-2", "group": "spatie/schema-org", "label": 0, "pair": "888e955fb6cfb4fd", "unit": "7b62415d17569924d16238a4aa28ab1af2db6561f56be9f59d8075fd1bc4a941"}
{"family": "injection", "fold": "outer-3", "group": "strangerstudios/paid-memberships-pro", "label": 1, "pair": "", "unit": "ac905d12b92954263c046bd0138a682cea86cf8476336d6baa664ade28d34ac6"}
{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 1, "pair": "218a45d869bd7e39", "unit": "309ebac468b7ab054c8396ed495770385ae3590caba43709745ba2d1db4b6d54"}
{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 1, "pair": "", "unit": "3cc8613813dec13cb1b7b45b296688da883f0b36cd2c4fc1da6e81703fb59ab9"}
{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "218a45d869bd7e39", "unit": "9bd2331dbc260451355307e4c37bae17ad6676654c66181934e1b35fb174cc5a"}
{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "b087ac4d0989e0b72698a9644ef4eac8073b4a15bb21730d63e4730b5cc58a60"}
{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "c5d1e8d19560c997018fdca12eb60f1799be76c33be058b88472f5a2fc6709be"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "22befc17a75a688e2f9358d2ddf9a7b197da44cd95dbdeae5e0191b113ee54d3"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "63f496d9ff53e39b", "unit": "6a98f8145776c78793b37156e9a81757841bb6de80016ab4fa575209d724db3f"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 1, "pair": "63f496d9ff53e39b", "unit": "7ac60c96c15f8517e089cc2240aedcf1fc30e54a41e35fe03c6b965bc393fbd6"}
{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "ae39cb55ef2a87c5347e0ddc40975f142450e82de4909e93ba62a7ed0efb4134"}
{"family": "injection", "fold": "outer-4", "group": "thenables/thenify", "label": 1, "pair": "d1b19898f8deaa41", "unit": "3763c7b407ad471820b5f8fa25bde0ba6d91e15c07c56681de4b41add418b0b1"}
{"family": "injection", "fold": "outer-4", "group": "thenables/thenify", "label": 0, "pair": "d1b19898f8deaa41", "unit": "bc8b934938c30687f3d4f982392d2cfa9504a916b463ccb5c46fe8a30eede488"}
{"family": "path", "fold": "outer-1", "group": "thumbor/thumbor", "label": 1, "pair": "f5f33fa6fe7b8492", "unit": "938bbf146a9037e687190bcee82c047492dffbe90efea40a49a647f81a0f1ed0"}
{"family": "path", "fold": "outer-1", "group": "thumbor/thumbor", "label": 0, "pair": "f5f33fa6fe7b8492", "unit": "b90c9f7e0011685f1a8e853a9ed0ae37fd437ef6d6eb0ba17c240aec2c22b603"}
{"family": "deserialization", "fold": "outer-3", "group": "tomasnorre/crawler", "label": 0, "pair": "f74364dc8e35c8ba", "unit": "11e760e99c5c852676c9288e18a250febb3854e9047b51e8b9ecc9f3b423b6f5"}
{"family": "deserialization", "fold": "outer-3", "group": "tomasnorre/crawler", "label": 1, "pair": "f74364dc8e35c8ba", "unit": "5fc980194040f4c4d803075d4085618cc0260ae8f3454b26b9b704d30200ffea"}
{"family": "output_encoding", "fold": "outer-3", "group": "tornadoweb/tornado", "label": 0, "pair": "db1c51833994ec55", "unit": "c2ce8320c778dd55434d7413844797b68e6fcabc4dc509458a9d60ca70b88c60"}
{"family": "output_encoding", "fold": "outer-3", "group": "tornadoweb/tornado", "label": 1, "pair": "db1c51833994ec55", "unit": "f2d6843547af1276ae8a4dff3a7786913fa79dc69540d4fbb95c4c091e27c7f2"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "truelayer/truelayer-dotnet", "label": 1, "pair": "f4690679bf9d80aa", "unit": "1a4e24df8d08672ecde4709a77ea5c437b2198fecff8bfff373b63c9eaed6e63"}
{"family": "untrusted_destination", "fold": "outer-1", "group": "truelayer/truelayer-dotnet", "label": 0, "pair": "f4690679bf9d80aa", "unit": "78711973ec84867671314dd7878769092b32f59860eafed50a0f6979daee76fc"}
{"family": "injection", "fold": "outer-2", "group": "ultimatemember/ultimatemember", "label": 0, "pair": "85c15e3fc2b370b2", "unit": "0990d8f41cb07c1cf217812e51a7025381de4ba68d1be3799e26241d81dcea67"}
{"family": "injection", "fold": "outer-2", "group": "ultimatemember/ultimatemember", "label": 1, "pair": "85c15e3fc2b370b2", "unit": "9da1d63f0c4da270f66eb87bf9c342ace883a0aef78c4643e254159b5da8fce3"}
{"family": "output_encoding", "fold": "outer-0", "group": "viewcomponent/view_component", "label": 0, "pair": "8f5a6abedd742bbe", "unit": "71e3ced643a244100cc3fde234f5b4b27655214c7ae0ff67ab61e3e48a8fdf78"}
{"family": "output_encoding", "fold": "outer-0", "group": "viewcomponent/view_component", "label": 1, "pair": "8f5a6abedd742bbe", "unit": "7c8cbfae87127042f9f43621345b4f7958bde27a3ff9db76254063e80735f3ce"}
{"family": "injection", "fold": "outer-1", "group": "vincentmorneau/apex-publish-static-files", "label": 1, "pair": "c9538342f2d7efca", "unit": "2b8db57d65ef1d67c457ff5a336e8a542d03387395d69d5e16815c4f00f40723"}
{"family": "injection", "fold": "outer-1", "group": "vincentmorneau/apex-publish-static-files", "label": 0, "pair": "c9538342f2d7efca", "unit": "e2ed113dc21ad5f7caa71ba4a5ece8f8f900a97fd0d00dc85f7ac79ff5bed19a"}
{"family": "injection", "fold": "outer-1", "group": "vvo/node-whereis", "label": 1, "pair": "8acd426c5771b37c", "unit": "983e146a60e061254b15dc8dd573cbba57f8dd0358f23ea1b9c313ef69069af5"}
{"family": "injection", "fold": "outer-1", "group": "vvo/node-whereis", "label": 0, "pair": "8acd426c5771b37c", "unit": "f80dfec45231e4778078cb10f00128ed445177cac0a062fc2df5ced5c0171f21"}
{"family": "deserialization", "fold": "outer-4", "group": "waterfutures/epyt-flow", "label": 0, "pair": "9b84800ce5c395cb", "unit": "9b7dfdceae03fd86ad3260a03f85bcdcff8f32f10c265d23743ff8dc98eeefb8"}
{"family": "deserialization", "fold": "outer-4", "group": "waterfutures/epyt-flow", "label": 1, "pair": "9b84800ce5c395cb", "unit": "b99dda150c0da3defa5f4c527ef6c402f5589dfb2f2f03f3e5e424910eb48c53"}
{"family": "path", "fold": "outer-4", "group": "webpack/webpack-dev-middleware", "label": 1, "pair": "ad64e8fac4ac3ee0", "unit": "36dfd01915a553d86576e8ecd3b737b8e2b297aa542f58cc83b448060e74398a"}
{"family": "path", "fold": "outer-4", "group": "webpack/webpack-dev-middleware", "label": 0, "pair": "ad64e8fac4ac3ee0", "unit": "c2b6d40f130d35bbfc16fab9411c664fa06321c59fb1d901a1987a69cb09c72e"}
{"family": "access_control", "fold": "outer-4", "group": "wintercms/winter", "label": 0, "pair": "d48c08b19f560848", "unit": "03d02c9e25eec0f380b125279fce35e8867065bf6e1ec75f9f39729ce6efdc77"}
{"family": "access_control", "fold": "outer-4", "group": "wintercms/winter", "label": 1, "pair": "d48c08b19f560848", "unit": "abbea69b7d0fa68c3ace442c25597e128cde354b417787b5913fd10d666c0d19"}
{"family": "output_encoding", "fold": "outer-3", "group": "xyproto/algernon", "label": 0, "pair": "3844ed3d703d7ef4", "unit": "445efda074e4d9f6c1634225eb08a3c3eb1257311c995ce891df2c791a6f5c8b"}
{"family": "output_encoding", "fold": "outer-3", "group": "xyproto/algernon", "label": 1, "pair": "3844ed3d703d7ef4", "unit": "8ac27b08549353ada819caaf99a5180d2f2f515b03aa8cd99a5d37b527ea3a5c"}
{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 1, "pair": "a63eae7d621a985e", "unit": "04ca0b583b09c3b4f67bd2496a4607ecb134d24b7bdaf8960fa151e40ebcc447"}
{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 0, "pair": "a63eae7d621a985e", "unit": "ecfa2a67726ca9d3e2621a4d8be60145095b76aa18ecf0268212341d63752871"}
{"family": "deserialization", "fold": "outer-4", "group": "zumba/json-serializer", "label": 0, "pair": "0d9f529b7f0084b6", "unit": "35aa369236f510634b3aa7b54b04e9da0f80877e515786a888e760f8fe33c524"}
{"family": "deserialization", "fold": "outer-4", "group": "zumba/json-serializer", "label": 1, "pair": "0d9f529b7f0084b6", "unit": "ef4f8613c0914eb815504e00cd6ee05df38352fd3b27826798e6623594017042"}
plane/experiments/exp-003-precision-bound-blocking.yaml
+65
View File
@@ -0,0 +1,65 @@
# Experiment exp-003 (learned-decision-engine Req 6, Req 7.3/7.4): the BLOCK rule. Registered before it runs;
# `ousast learn experiment register` records this file's sha256, and a changed file is a new experiment.
# kind: rule -- both arms decide from the SAME scores: one scoring pass of the six slice-2 incumbent programs, replayed
# from the response cache only (zero cost by construction: a zero-ceiling meter around a refusing client; a unit with any
# uncached response is excluded and counted, never asked; the meter's calls and usd must read 0 or the run is void).
#
# Threshold selection (arm B), fixed here: per family and per outer fold j, the candidate thresholds are the raw scores
# s of the training rows, i.e. the family's paired units in the OTHER four outer folds (each of those predictions is
# itself out of repository: retrieval and demonstrations never cross its own fold's boundary). The chosen threshold is
# the lowest t such that, among those training rows with s >= t whose majority verdict is not `unsure`, the Wilson 95%
# lower bound of precision is >= 0.95. No such t -> fold j gets no BLOCK. Fold j's units are then flagged when not
# `unsure` and s >= t. Fold j's labels never enter its own threshold (tests/test_learn_rules.py).
#
# Pre-registered expectation (power): a Wilson lower bound of 0.95 needs at least 73 flagged units with no false flag.
# The frozen units (353 in 146 repository groups, 244 paired) hold 38 paired positives for injection and 16-19 for each
# other family, all five folds together; the training rows of any one fold hold fewer. So arm B cannot offer a BLOCK in
# any family by construction, and the expected verdict is inconclusive with arm B flagging nothing. That is the finding
# this run would confirm: on this data the bound, not the calibration gate, is what makes BLOCK unreachable. Arm C (point precision >= 0.95 over >= 10 training flags) is a secondary,
# non-deciding arm that measures how far the held-out precision is from the bar when the bound is relaxed.
id: exp-003-precision-bound-blocking
hypothesis: >
The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4,
which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training
folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision
bar on held-out repositories where the calibrated rule offers no BLOCK.
kind: rule
programs: # the slice-2 incumbents (benchmarks/measurements/2026-10-02-decision-engine-slice-2, program.id)
access_control: 745cb1d76bbb09dc1574d333d2118489be62e39886517894d3699fa565b0e8c2
deserialization: 3c0c2831be7d78198a62351661c37867363c501cb40e3ed7b0119c155af09654
injection: b207a0710e0abf053b3b9397b25f9ee1cab4679144865e3a82c1b9be2c7b4982
output_encoding: 5d768f16724562c381c09949ff3df9cbb11d58cbb7a9c0515d0dc3f858d3818e
path: aea396680a4874dcb6de70e240ca0480c83424debdd3710bc3f1ea6a612661f3
untrusted_destination: 1242b3973fc89056aa08e00843043d68235df217d8e3ef8a0ea0e0b81c142753
arms:
A:
label: today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p
rule: {name: calibrated_block}
B:
label: selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95
rule: {name: precision_bound_block, bound: wilson, precision: 0.95, select_on: paired}
C:
label: secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags
rule: {name: precision_bound_block, bound: point, precision: 0.95, min_flagged: 10, select_on: paired}
unit: candidate
pairing: paired # every unit scored once; every arm decides every unit
cluster: repository
order: {randomise: none, seed: 3}
# primary population: paired units (both sides of the candidate present). It excludes every positives-only group and any
# unpaired unit, whose positives never had to be told apart from their own fix. Pooled (every unit) is reported beside it.
populations: {primary: paired, secondary: [pooled]}
metric:
primary:
- {name: flagged_precision_lb, target: 0.95} # held-out (outer-fold) Wilson 95% LB of flagged precision, per family, arm B, paired
secondary: [flagged_precision, coverage, flagged] # coverage = flagged true positives / positives
# Decision rule (decide_rule): REJECT B if in any family B flags held-out paired units at a point precision < 0.95;
# else ADOPT B if in some family B's held-out paired Wilson LB >= 0.95; else INCONCLUSIVE (the incumbent rule stays).
# A family "reaches the M4 precision bar on held-out repositories" when its held-out paired Wilson LB >= 0.95 (the
# point precision is reported beside it).
decision: {arm: B, target_precision: 0.95}
test: {resamples: 2000} # repository-cluster bootstrap of the held-out flagged precision, beside the Wilson bound
mde: 0.0 # not a paired difference test; see the power note above
budget_usd: {total: 0} # zero cost is a hard rule
families: [access_control, deserialization, injection, output_encoding, path, untrusted_destination]
folds: {kind: outer, k: 5, seed: 0, compile_fraction: 0.25} # the incumbent evaluation's folds (learn/compile.toml seed)
units_file: plane/experiments/exp-003-precision-bound-blocking.units.jsonl # frozen by `ousast learn experiment units`
src/openultrasast/learn/cli.py
+19 -7
View File
@@ -122,11 +122,17 @@ def _experiment(args: argparse.Namespace) -> int:
print(json.dumps({k: row[k] for k in shown}, indent=2))
return 0
if args.experiment_command == "analyse":
row = ex.check_registered(store, manifest, need_units=False)
row = ex.check_registered(store, manifest, need_units=manifest.kind == "rule")
provenance = {"code_commit": ex.committed_at_head(manifest.path)}
report = ex.analyse(manifest, ex.load_outcomes(store, manifest.id), resamples=args.resamples, provenance=provenance)
outcomes = ex.load_outcomes(store, manifest.id)
if manifest.kind == "rule":
units = ex.read_units(manifest.units_file)
report = ex.analyse_rule(manifest, outcomes, units, resamples=args.resamples, provenance=provenance)
else:
report = ex.analyse(manifest, outcomes, resamples=args.resamples, provenance=provenance)
if args.record:
store.put_row(ex.result_row(manifest, str(row["pin"]), report))
result = ex.rule_result_row if manifest.kind == "rule" else ex.result_row
store.put_row(result(manifest, str(row["pin"]), report))
_write(report, args.out or (ex.RESULT_DIR / manifest.id / "result.json" if args.record else None))
return 0
from .embeddings import EmbeddingCache
@@ -143,6 +149,16 @@ def _experiment(args: argparse.Namespace) -> int:
cache = EmbeddingCache(store)
vectors = {sha: v for sha in cache.names() if (v := cache.get(sha)) is not None} or None
parameters = _model_parameters(args.manifest_model)
def excerpt_text(sha: str) -> str | None:
data = store.get_blob("excerpts", sha)
return data.decode("utf-8") if data is not None else None
if manifest.kind == "rule": # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved
caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store)
summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors)
_write(summary.as_dict(), args.out)
return 0
callers: dict[str, Caller] = {}
for name, arm in manifest.arms.items():
if arm.replay_only:
@@ -157,10 +173,6 @@ def _experiment(args: argparse.Namespace) -> int:
client = MeteredClient(resolved[0], prices=parameters, budget_usd=arm.budget_usd)
callers[name] = Caller(client, _arm_model(manifest, arm, store), parameters, store=store)
def excerpt_text(sha: str) -> str | None:
data = store.get_blob("excerpts", sha)
return data.decode("utf-8") if data is not None else None
early = not args.no_early_stop
summary = ex.run(store, manifest, memory, callers, excerpt_text, vectors, stop_early=early, resamples=args.resamples)
except ex.ExperimentError as exc:
src/openultrasast/learn/evaluate.py
+86 -3
View File
@@ -54,6 +54,7 @@ class Prediction:
parse_failed: int = 0
p: float | None = None
point: str | None = None # BLOCK | ADVISORY | DROP, from points chosen on the other folds
pair: str = "" # the two sides of one candidate share it; "" when only one side is present
@property
def blockable(self) -> bool:
@@ -128,6 +129,87 @@ def nested_points(preds: Sequence[Prediction]) -> tuple[list[Prediction], dict[s
return out, points
BOUNDS = ("wilson", "point")
def precision_threshold(
rows: Sequence[Prediction], *, precision: float = TARGET_PRECISION, bound: str = "wilson", min_flagged: int = 1
) -> float | None:
"""The lowest raw-score threshold ``t`` at which the blockable rows with ``s >= t`` reach ``precision``: their
Wilson 95% lower bound (``bound="wilson"``) or their point precision over at least ``min_flagged`` rows
(``bound="point"``). Candidate thresholds are the rows' own scores; ``None`` when no threshold qualifies."""
if bound not in BOUNDS:
raise ValueError(f"bound {bound!r} (one of {', '.join(BOUNDS)})")
blockable = [r for r in rows if r.blockable]
for t in sorted({r.s for r in blockable}):
chosen = [r for r in blockable if r.s >= t]
hits = sum(r.label for r in chosen)
if len(chosen) < max(1, min_flagged):
continue
level = wilson_lower(hits, len(chosen)) if bound == "wilson" else hits / len(chosen)
if level >= precision:
return t
return None
def nested_precision_block(
preds: Sequence[Prediction],
*,
eligible: Callable[[Prediction], bool] | None = None,
precision: float = TARGET_PRECISION,
bound: str = "wilson",
min_flagged: int = 1,
) -> tuple[list[Prediction], dict[str, dict[str, float | None]]]:
"""Selective blocking on the raw score, per family: the threshold for outer fold j is chosen on the *other* folds'
rows of the family only (and, with ``eligible``, only those rows); the held-out fold's labels never enter it.
Every row of fold j is then decided with that threshold: ``BLOCK`` when blockable and ``s >= t``, else ``DROP``.
A family whose training folds reach no threshold offers no BLOCK on that fold. Returns the decided rows and the
thresholds by family and fold."""
thresholds: dict[str, dict[str, float | None]] = {}
for family in sorted({r.family for r in preds}):
rows = [r for r in preds if r.family == family]
thresholds[family] = {
fold: precision_threshold(
[r for r in rows if r.fold != fold and (eligible is None or eligible(r))],
precision=precision, bound=bound, min_flagged=min_flagged,
)
for fold in sorted({r.fold for r in rows})
} # fmt: skip
out = []
for r in preds:
t = thresholds[r.family][r.fold]
out.append(replace(r, point="BLOCK" if t is not None and r.blockable and r.s >= t else "DROP"))
return out, thresholds
def calibrated_block(preds: Sequence[Prediction], *, seed: int = 0, **floors: int) -> tuple[list[Prediction], dict[str, Any]]:
"""Today's BLOCK rule, per family as the slice-2 evaluation applied it: the Platt map cross-fitted on the other
folds, the calibration check on the family's held-out calibrated predictions, the BLOCK point chosen on the other
folds' calibrated predictions (nested); a row is ``BLOCK`` only when the family's calibration holds. Returns the
decided rows and, per family, the check and the calibrated thresholds by fold."""
out: list[Prediction] = []
info: dict[str, Any] = {}
for family in sorted({r.family for r in preds}):
rows = [r for r in preds if r.family == family]
ps, _ = cross_fit([Row(r.fold, r.family, r.group, r.label, r.s) for r in rows], **floors)
calibrated = [replace(r, p=p) for r, p in zip(rows, ps, strict=True)]
decided = [r for r in calibrated if r.p is not None]
checked = (
check([float(r.p) for r in decided if r.p is not None], [r.label for r in decided], [r.group for r in decided], seed=seed)
if len({r.label for r in decided}) == 2
else None
)
holds = bool(checked and checked.holds)
nested, points = nested_points(calibrated)
out += [replace(r, point="BLOCK" if holds and r.point == "BLOCK" else "DROP") for r in nested]
info[family] = {
"calibration_holds": holds, "ece": None if checked is None else round(checked.ece, 4),
"thresholds_by_fold": {f: p.block for f, p in points.items()},
"reason_by_fold": {f: ("calibration" if not holds else p.block_reason) for f, p in points.items()},
} # fmt: skip
return out, info
def _at(rows: Sequence[Prediction], level: str) -> dict[str, float | None]:
flagged = [r for r in rows if r.point in (("BLOCK",) if level == "block" else ("BLOCK", "ADVISORY"))]
positives = sum(r.label for r in rows)
@@ -326,8 +408,9 @@ def candidate_from_example(
__all__ = [
"PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "candidate_from_example", "choose_points",
"cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible", "learning_curve", "nested_points", "predict", "strata",
"summary", "targets", "unevaluable_families",
"BOUNDS", "PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "calibrated_block",
"candidate_from_example", "choose_points", "cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible",
"learning_curve", "nested_points", "nested_precision_block", "precision_threshold", "predict", "strata", "summary", "targets",
"unevaluable_families",
"wilson_lower",
] # fmt: skip
src/openultrasast/learn/experiments.py
+313 -10
View File
@@ -41,10 +41,10 @@ from typing import Any
import yaml
from ..plane.budget import BudgetExhausted
from ..plane.budget import BudgetExhausted, MeteredClient
from ..plane.memory import MemoryStore, row_id
from .compile import load_program, spec_of
from .evaluate import candidate_from_example
from .evaluate import BOUNDS, TARGET_PRECISION, Prediction, calibrated_block, candidate_from_example, nested_precision_block, wilson_lower
from .examples import EXAMPLE_KIND, Example
from .folds import Fold, compile_split, folds_digest, outer_folds
from .program import Caller, Decision, Program, ProgramSpec
@@ -57,7 +57,15 @@ RESULT_KIND = "experiment_result"
REGISTER_TASK = "register"
POPULATION = "experiments" # the envelope of every experiment row; `split` is the experiment id
RUN_TASK = "run"
ARM_KINDS = ("program", "plane")
ARM_KINDS = ("program", "plane", "rule")
# decision-rule experiments (kind: rule): one replay-only scoring pass of the incumbent programs, arms differ only in
# the BLOCK rule applied to the same scores
RULES = ("calibrated_block", "precision_bound_block")
POPULATIONS = ("paired", "pooled") # paired: units whose candidate has both sides present; pooled: every unit
SCORE_ARM = "score" # the arm name of a rule experiment's outcome rows (the shared scores)
MISS_VERDICT = "replay_miss" # an outcome row of a unit excluded because a response was not cached
# rule metrics, per family and population of held-out (outer-fold) units: direction only (no paired test)
RULE_METRICS: Mapping[str, int] = {"flagged_precision_lb": 1, "flagged_precision": 1, "coverage": 1, "flagged": 1}
UNITS = ("candidate", "file", "repository")
PAIRINGS = ("paired", "assigned")
# O'Brien-Fleming boundaries for two equally spaced looks at alpha 0.05, two-sided (design section 6).
@@ -96,6 +104,7 @@ class Arm:
replay_only: bool = False
repeats: int = 1
budget_usd: float = 0.0
rule: Mapping[str, Any] = field(default_factory=dict) # rule arms: {name: calibrated_block | precision_bound_block, ...}
@dataclass(frozen=True)
@@ -119,11 +128,11 @@ class Metric:
@property
def direction(self) -> int:
return METRICS[self.name][0]
return RULE_METRICS[self.name] if self.name in RULE_METRICS else METRICS[self.name][0]
@property
def binary(self) -> bool:
return METRICS[self.name][1]
return self.name in METRICS and METRICS[self.name][1]
def as_dict(self) -> dict[str, Any]:
return {k: v for k, v in asdict(self).items() if v is not None}
@@ -198,6 +207,7 @@ def load_manifest(path: Path) -> Manifest:
bool(spec.get("replay_only", False)),
int(spec.get("repeats", 1)),
float(spec.get("budget_usd", 0.0)),
dict(spec.get("rule") or {}),
) # fmt: skip
for name, spec in dict(raw["arms"]).items()
}
@@ -210,9 +220,12 @@ def load_manifest(path: Path) -> Manifest:
primary_raw = metric["primary"]
primary = tuple(Metric.parse(m) for m in (primary_raw if isinstance(primary_raw, list) else [primary_raw]))
secondary = tuple(Metric.parse(m) for m in metric.get("secondary", []))
known = RULE_METRICS if kind == "rule" else METRICS
for m in (*primary, *secondary):
if m.name not in METRICS:
raise ExperimentError(f"{path}: metric {m.name!r} (one of {', '.join(METRICS)})")
if m.name not in known:
raise ExperimentError(f"{path}: metric {m.name!r} (one of {', '.join(known)})")
if kind == "rule":
_check_rule_manifest(path, raw, arms)
test = dict(raw.get("test") or {})
stopping = dict(raw.get("stopping") or {"looks": [1.0], "boundaries": [1.959964]})
looks = tuple(float(x) for x in stopping.get("looks", [0.5, 1.0]))
@@ -244,11 +257,39 @@ def load_manifest(path: Path) -> Manifest:
raise ExperimentError(f"{path}: missing {exc}") from exc
if manifest.id != path.stem:
raise ExperimentError(f"{path}: id {manifest.id!r} must equal the file stem")
if kind == "program" and not manifest.families:
raise ExperimentError(f"{path}: a program experiment names its families")
if kind in ("program", "rule") and not manifest.families:
raise ExperimentError(f"{path}: a {kind} experiment names its families")
if kind == "rule" and set(manifest.families) - set(manifest.programs):
raise ExperimentError(f"{path}: a rule experiment names a program for every family (programs:)")
return manifest
def _check_rule_manifest(path: Path, raw: Mapping[str, Any], arms: Mapping[str, Arm]) -> None:
"""A decision-rule experiment costs nothing by construction: no arm overrides the program, no budget, and every
arm names a known rule with known parameters; the populations are declared before the run."""
budget = raw["budget_usd"]
total = float(budget["total"] if isinstance(budget, Mapping) else budget)
if total != 0 or any(a.budget_usd for a in arms.values()):
raise ExperimentError(f"{path}: a rule experiment replays cached responses only: budget_usd must be 0")
for arm in arms.values():
if arm.spec or arm.task_env or arm.programs:
raise ExperimentError(f"{path}: arm {arm.name} of a rule experiment may change the BLOCK rule only (rule:)")
name = arm.rule.get("name")
if name not in RULES:
raise ExperimentError(f"{path}: arm {arm.name} rule.name {name!r} (one of {', '.join(RULES)})")
allowed = {"name"} | ({"precision", "bound", "min_flagged", "select_on"} if name == "precision_bound_block" else set())
if set(arm.rule) - allowed:
raise ExperimentError(f"{path}: arm {arm.name} rule parameters {sorted(set(arm.rule) - allowed)} are not understood")
if arm.rule.get("bound", "wilson") not in BOUNDS or arm.rule.get("select_on", "paired") not in POPULATIONS:
raise ExperimentError(f"{path}: arm {arm.name} bound in {BOUNDS} and select_on in {POPULATIONS}")
populations = dict(raw.get("populations") or {})
if populations.get("primary") not in POPULATIONS or any(p not in POPULATIONS for p in populations.get("secondary", [])):
raise ExperimentError(f"{path}: populations.primary (and secondary) in {POPULATIONS}")
decision = dict(raw.get("decision") or {})
if not 0 < float(decision.get("target_precision", 0)) <= 1 or decision.get("arm") not in arms:
raise ExperimentError(f"{path}: decision.target_precision in (0, 1] and decision.arm naming an arm")
# --- register ------------------------------------------------------------------------------------------------------------
@@ -383,7 +424,7 @@ def program_folds(memory: Sequence[Example], manifest: Manifest) -> tuple[list[F
def freeze_units(store: MemoryStore, manifest: Manifest, memory: Sequence[Example]) -> list[Unit]:
"""The evaluation candidates of the manifest's families under the incumbent's outer folds, paired by candidate
across the two pins where both sides are in memory (no model; the list is then committed and registered)."""
if manifest.kind != "program":
if manifest.kind not in ("program", "rule"):
raise ExperimentError(f"{manifest.id}: units of a plane experiment come from candidate generation, not the memory")
folds, _ = program_folds(memory, manifest)
sides: dict[str, tuple[str, str, str]] = {}
@@ -812,6 +853,8 @@ def run(
"""Every unit in every arm, paired, repositories in the pre-shuffled look order, a seeded coin per unit for which
arm goes first; outcomes already in the store are not decided again. After the first look's repositories the
primary is tested against its boundary and, when crossed, the run stops. A ceiling ends the run ``unfinished``."""
if manifest.kind == "rule":
raise ExperimentError(f"{manifest.id}: a rule experiment scores once at zero cost (run_rule), its arms differ only in the rule")
if manifest.kind != "program":
raise ExperimentError(f"{manifest.id}: plane arms run as plane Runs; their outcomes arrive as arm_outcome rows")
row = check_registered(store, manifest)
@@ -892,7 +935,267 @@ def run(
return summary
# --- decision-rule experiments (kind: rule) -----------------------------------------------------------------------------
class ZeroCostViolation(ExperimentError):
"""A model call reached the client of a zero-cost run, or the meter recorded spend: the run is void."""
class _RefusingChat:
"""The inner client behind a zero-cost run's meter. The meter's zero ceilings stop every call before it gets here;
reaching it means a guard was bypassed."""
usage: list[Any] = []
def complete(self, **_: Any) -> Any:
raise ZeroCostViolation("a model call reached the client of a zero-cost run")
def zero_cost_caller(model: str, prices: Mapping[str, Any], store: MemoryStore | None) -> tuple[Caller, MeteredClient]:
"""A caller that answers only from the response cache: its client is a meter with zero ceilings (USD and calls)
around a client that refuses. A response that is not cached raises ``BudgetExhausted`` before any call; the run
excludes that unit and counts it. The meter is the proof: its ``calls`` and ``usd`` must stay 0."""
meter = MeteredClient(_RefusingChat(), prices=prices, budget_usd=0.0, budget_calls=0) # type: ignore[arg-type]
return Caller(meter, model, prices, store=store), meter
def _checked_units(manifest: Manifest, memory: Sequence[Example]) -> tuple[list[Unit], dict[str, Fold], str]:
units = read_units(manifest.units_file)
folds, fdigest = program_folds(memory, manifest)
by_name = {f.name: f for f in folds}
index = {e.id for e in memory}
for unit in units:
fold = by_name.get(unit.fold)
if fold is None or unit.group not in fold.eval_groups or unit.unit not in index:
raise ExperimentError(
f"{manifest.id}: unit {unit.unit[:12]} is not under fold {unit.fold!r} of this memory; the memory changed"
)
return units, by_name, fdigest
def run_rule(
store: MemoryStore,
manifest: Manifest,
memory: Sequence[Example],
caller: Caller,
meter: MeteredClient,
excerpt_text: Callable[[str], str | None],
vectors: Mapping[str, Sequence[float]] | None = None,
*,
on_decision: Callable[[Outcome, Decision], None] | None = None,
) -> RunSummary:
"""The one scoring pass of a rule experiment: every unit decided once by its family's incumbent program, answered
only from the response cache. A unit whose responses are not all cached is excluded and counted (an outcome row
with ``verdict = replay_miss`` and no score), never asked. Raises :class:`ZeroCostViolation` when the meter shows
a call or any spend afterwards."""
if manifest.kind != "rule":
raise ExperimentError(f"{manifest.id}: run_rule runs decision-rule experiments only")
if caller.client is not meter or meter.budget_usd != 0.0 or meter.budget_calls != 0:
raise ZeroCostViolation(f"{manifest.id}: a rule experiment runs only behind a zero-ceiling meter (zero_cost_caller)")
row = check_registered(store, manifest)
units, by_name, fdigest = _checked_units(manifest, memory)
index = {e.id: e for e in memory}
programs = {f: Program(arm_spec(manifest, manifest.arms["A"], f, store), memory, excerpt_text, vectors) for f in manifest.families}
models = {spec_of(load_program(store, manifest.programs[f])).model for f in manifest.families}
if models != {caller.model}:
raise ExperimentError(f"{manifest.id}: the programs' models {sorted(models)} differ from the caller's {caller.model!r}")
candidate_of = candidate_from_example(excerpt_text, vectors)
have = {outcome_id(manifest.id, o.arm, o.unit, o.replicate) for o in load_outcomes(store, manifest.id)}
decided, skipped = {SCORE_ARM: 0}, {"already": 0, MISS_VERDICT: 0}
fold_seed = int(manifest.folds.get("seed", 0))
for unit in sorted(units, key=lambda u: (u.group, u.family, u.unit)):
if outcome_id(manifest.id, SCORE_ARM, unit.unit, 0) in have:
skipped["already"] += 1
continue
candidate = candidate_of(index[unit.unit])
if candidate is None:
raise ExperimentError(f"{manifest.id}: unit {unit.unit[:12]} has no excerpt in the store")
try:
decision = programs[unit.family].decide(candidate, by_name[unit.fold], caller, seed=fold_seed)
except BudgetExhausted: # a response is not cached: the zero ceiling stopped the call before it started
outcome = Outcome(unit.unit, unit.group, unit.family, unit.label, unit.pair, SCORE_ARM, 0, None, MISS_VERDICT, 0.0)
skipped[MISS_VERDICT] += 1
store.put_row(outcome_row(manifest.id, str(row["pin"]), outcome))
continue
outcome = Outcome(unit.unit, unit.group, unit.family, unit.label, unit.pair, SCORE_ARM, 0, decision.s, decision.verdict, 0.0,
decision.parse_failed) # fmt: skip
store.put_row(outcome_row(manifest.id, str(row["pin"]), outcome))
decided[SCORE_ARM] += 1
if on_decision is not None:
on_decision(outcome, decision)
if meter.calls or (meter.usd or 0.0) > 0 or caller.calls:
raise ZeroCostViolation(f"{manifest.id}: the meter recorded {meter.calls} calls and ${meter.usd}; the run is void")
spend = {SCORE_ARM: {"usd": meter.usd, "client_calls": meter.calls, "replayed": caller.replayed, "cache_digest": caller.cache_digest(),
"replay_priced_usd": caller.usd()}} # fmt: skip
summary = RunSummary(manifest.id, "done", len(units), decided, skipped, spend, fdigest)
store.put_row({
"id": row_id(EXPERIMENT_KIND, f"experiment-{manifest.id}", RUN_TASK, manifest.id), "kind": EXPERIMENT_KIND,
"repo": experiment_repo(manifest.id), "pin": str(row["pin"]), "run": f"experiment-{manifest.id}", "task": RUN_TASK,
"population": POPULATION, "split": manifest.id, "image": "host", "experiment": manifest.id,
"manifest_digest": manifest.digest, **summary.as_dict(),
}) # fmt: skip
return summary
def min_flagged_for_bound(target: float, z: float = 1.959964) -> int:
"""The fewest flagged units, all correct, whose Wilson lower bound reaches ``target`` (73 at 0.95)."""
n = 1
while wilson_lower(n, n, z) < target:
n += 1
return n
def _group_bootstrap_precision(rows: Sequence[Prediction], *, resamples: int, seed: int) -> list[float] | None:
by_group: dict[str, list[Prediction]] = {}
for r in rows:
by_group.setdefault(r.group, []).append(r)
names = sorted(by_group)
rng = random.Random(seed)
values = []
for _ in range(resamples if names else 0):
sample = [r for g in (rng.choice(names) for _ in names) for r in by_group[g]]
flagged = [r for r in sample if r.point == "BLOCK"]
if flagged:
values.append(sum(r.label for r in flagged) / len(flagged))
if len(values) < 2:
return None
values.sort()
return [values[int(0.025 * (len(values) - 1))], values[int(0.975 * (len(values) - 1))]]
def flag_summary(rows: Sequence[Prediction], target: float, *, resamples: int = 2000, seed: int = 0) -> dict[str, Any]:
"""Held-out BLOCK decisions of one population: flagged count, flagged precision with its Wilson 95% lower bound
and a repository-cluster bootstrap interval, coverage (the share of positives flagged), and the M4 reading."""
flagged = [r for r in rows if r.point == "BLOCK"]
hits = sum(r.label for r in flagged)
positives = sum(r.label for r in rows)
precision = hits / len(flagged) if flagged else None
lower = wilson_lower(hits, len(flagged)) if flagged else None
return {
"units": len(rows), "groups": len({r.group for r in rows}), "positives": positives, "flagged": len(flagged),
"true_flagged": hits, "flagged_precision": None if precision is None else round(precision, 4),
"flagged_precision_lb": None if lower is None else round(lower, 4),
"flagged_precision_ci": _group_bootstrap_precision(rows, resamples=resamples, seed=seed) if flagged else None,
"coverage": round(hits / positives, 4) if positives else None,
"meets_target_point": precision is not None and precision >= target,
"meets_target_lb": lower is not None and lower >= target,
} # fmt: skip
def _apply_rule(arm: Arm, preds: Sequence[Prediction], seed: int) -> tuple[list[Prediction], dict[str, Any]]:
if arm.rule["name"] == "calibrated_block":
return calibrated_block(preds, seed=seed)
select_on = str(arm.rule.get("select_on", "paired"))
decided, thresholds = nested_precision_block(
preds, eligible=(lambda r: bool(r.pair)) if select_on == "paired" else None,
precision=float(arm.rule.get("precision", TARGET_PRECISION)), bound=str(arm.rule.get("bound", "wilson")),
min_flagged=int(arm.rule.get("min_flagged", 1)),
) # fmt: skip
return decided, {f: {"thresholds_by_fold": t} for f, t in thresholds.items()}
def decide_rule(manifest: Manifest, families: Mapping[str, Mapping[str, Any]]) -> tuple[str, str]:
"""The pre-registered rule: **reject** the decision arm when it flags held-out units of the primary population in
some family at a point precision below the target; **adopt** it when it does not and some family's held-out
Wilson lower bound reaches the target; otherwise **inconclusive** (the incumbent rule stays)."""
decision = dict(manifest.raw["decision"])
arm, target = str(decision["arm"]), float(decision["target_precision"])
primary = str(manifest.raw["populations"]["primary"])
cells = {f: v[arm][primary] for f, v in families.items()}
low = sorted(f for f, c in cells.items() if c["flagged"] and not c["meets_target_point"])
if low:
return "reject", f"{arm} flags held-out {primary} units below precision {target} in {', '.join(low)}"
reached = sorted(f for f, c in cells.items() if c["meets_target_lb"])
if reached:
return "adopt", f"{arm}'s held-out Wilson lower bound reaches {target} in {', '.join(reached)} and nowhere flags below it"
if not any(c["flagged"] for c in cells.values()):
return "inconclusive", f"{arm} offers no BLOCK on held-out {primary} units in any family; the incumbent stays"
return "inconclusive", f"{arm} flags only at a Wilson lower bound below {target}; the incumbent stays"
def analyse_rule(
manifest: Manifest,
outcomes: Sequence[Outcome],
units: Sequence[Unit],
*,
resamples: int | None = None,
provenance: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Counts and intervals only: every arm's rule applied to the same held-out scores, per family and population,
the pooled-over-families totals, the exclusions, the training capacity of each family against the Wilson bound,
and the pre-registered verdict."""
n = resamples if resamples is not None else min(manifest.resamples, 2000)
target = float(manifest.raw["decision"]["target_precision"])
populations = [str(manifest.raw["populations"]["primary"]), *[str(p) for p in manifest.raw["populations"].get("secondary", [])]]
by_unit = {u.unit: u for u in units}
scores = {o.unit: o for o in outcomes if o.arm == SCORE_ARM and o.replicate == 0 and o.unit in by_unit}
preds = [
Prediction(by_unit[u].fold, o.family, o.group, o.label, float(o.s), o.verdict, parse_failed=o.parse_failed, pair=o.pair)
for u, o in sorted(scores.items())
if o.s is not None
] # fmt: skip
excluded = {f: sum(1 for o in scores.values() if o.s is None and o.family == f) for f in manifest.families}
not_run = {f: sum(1 for u in units if u.family == f and u.unit not in scores) for f in manifest.families}
def select(rows: Sequence[Prediction], population: str) -> list[Prediction]:
return [r for r in rows if r.pair] if population == "paired" else list(rows)
decided = {name: _apply_rule(arm, preds, manifest.seed) for name, arm in manifest.arms.items()}
families: dict[str, dict[str, Any]] = {}
for family in manifest.families:
cell: dict[str, Any] = {}
for name, (rows, info) in decided.items():
mine = [r for r in rows if r.family == family]
cell[name] = {**info.get(family, {}), **{p: flag_summary(select(mine, p), target, resamples=n, seed=manifest.seed)
for p in populations}} # fmt: skip
mine = [r for r in preds if r.family == family]
folds = sorted({r.fold for r in mine})
cell["capacity"] = {
"evaluated": len(mine), "excluded_replay_miss": excluded[family], "not_run": not_run[family],
"paired_positives_by_training_folds": {f: sum(r.label for r in mine if r.pair and r.fold != f) for f in folds},
} # fmt: skip
families[family] = cell
overall = {
name: {p: flag_summary(select(rows, p), target, resamples=n, seed=manifest.seed) for p in populations}
for name, (rows, _) in decided.items()
}
verdict, reason = decide_rule(manifest, families)
spend = {SCORE_ARM: round(sum(o.usd or 0.0 for o in outcomes if o.arm == SCORE_ARM), 6)}
return {
"experiment": manifest.id, "manifest_digest": manifest.digest, "hypothesis": manifest.hypothesis,
"arms": {a: {"label": arm.label, "rule": dict(arm.rule)} for a, arm in manifest.arms.items()},
"units": {"frozen": len(units), "evaluated": len(preds), "excluded_replay_miss": sum(excluded.values()),
"not_run": sum(not_run.values()), "paired_evaluated": sum(1 for r in preds if r.pair),
"groups": len({r.group for r in preds}), "by_label": {str(lab): sum(r.label == lab for r in preds) for lab in (0, 1)},
"unsure": sum(1 for r in preds if not r.blockable)},
"populations": {"primary": populations[0], "secondary": populations[1:]}, "target_precision": target,
"min_flagged_for_bound": min_flagged_for_bound(target), "primary": [m.as_dict() for m in manifest.primary],
"secondary": [m.as_dict() for m in manifest.secondary], "families": families, "overall": overall, "resamples": n,
"decision": verdict, "reason": reason, "spend_usd": spend, **dict(provenance or {}),
} # fmt: skip
def rule_result_row(manifest: Manifest, pin: str, result: Mapping[str, Any]) -> dict[str, Any]:
"""The ``experiment_result`` row of a rule experiment, with the column types of :func:`result_row`."""
primary = result["populations"]["primary"]
tests = {
f: {a: {k: cell[a][primary][k] for k in ("flagged", "flagged_precision", "flagged_precision_lb", "coverage")}
for a in manifest.arms}
for f, cell in result["families"].items()
} # fmt: skip
return {
"id": row_id(RESULT_KIND, f"experiment-{manifest.id}", "analyse", manifest.id), "kind": RESULT_KIND,
"repo": experiment_repo(manifest.id), "pin": pin, "run": f"experiment-{manifest.id}", "task": "analyse", "population": POPULATION,
"split": manifest.id, "image": "host", "experiment": manifest.id, "manifest_digest": manifest.digest,
"decision": result["decision"], "reason": result["reason"], "unit_counts": result["units"], "spend_usd": result["spend_usd"],
"tests": tests, "looks_taken": [],
} # fmt: skip
__all__ = [
"MISS_VERDICT", "POPULATIONS", "RULES", "RULE_METRICS", "SCORE_ARM", "ZeroCostViolation", "analyse_rule", "decide_rule",
"flag_summary", "min_flagged_for_bound", "rule_result_row", "run_rule", "zero_cost_caller",
"ALPHA", "ARM_KINDS", "EXPERIMENT_KIND", "MANIFEST_DIR", "METRICS", "OBF_TWO_LOOKS", "OUTCOME_KIND", "RESAMPLES", "RESULT_DIR",
"RESULT_KIND", "TOST_MARGIN", "VERDICTS", "Arm", "ArmUnit", "Estimate", "ExperimentError", "Look", "Manifest", "McNemar", "Metric",
"Outcome", "RunSummary", "Unit", "analyse", "arm_spec", "arm_units", "auc", "check_registered", "committed_at_head", "decide",
tests/test_learn_rules.py
+247
View File
@@ -0,0 +1,247 @@
"""Decision-rule experiments (kind: rule): selective blocking on the raw score, chosen on training folds only, against
today's calibrated BLOCK rule, scored once from the response cache at zero cost (exp-003)."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from test_learn_experiments import CODE, EXAMPLES, PRICES, _json_shape, artifact, committed, example_rows, excerpt_text, paid, put_program
from openultrasast.learn import experiments as ex
from openultrasast.learn.evaluate import Prediction, calibrated_block, nested_precision_block, precision_threshold, wilson_lower
from openultrasast.learn.program import Candidate, Program
from openultrasast.learn.retrieve import Target
from openultrasast.plane.memory import FileStore
ROOT = Path(__file__).resolve().parents[1]
RULE_MANIFEST = """
id: {id}
hypothesis: selective blocking beats the calibrated rule
kind: rule
programs: {{injection: {pid}, path: {pid}}}
arms:
A: {{label: today, rule: {{name: calibrated_block}}}}
B: {{label: bound, rule: {{name: precision_bound_block, bound: wilson, precision: 0.95, select_on: paired}}}}
C: {{label: point, rule: {{name: precision_bound_block, bound: point, precision: 0.95, min_flagged: 2, select_on: paired}}}}
populations: {{primary: paired, secondary: [pooled]}}
decision: {{arm: B, target_precision: 0.95}}
metric:
primary: [{{name: flagged_precision_lb, target: 0.95}}]
secondary: [flagged_precision, coverage, flagged]
test: {{resamples: 50}}
budget_usd: {{total: 0}}
families: [injection, path]
folds: {{k: 5, seed: 0, compile_fraction: 0.25}}
units_file: {units}
"""
def write_rule_manifest(tmp_path: Path, pid: str, name: str = "exp-901-rule") -> Path:
path = tmp_path / f"{name}.yaml"
path.write_text(RULE_MANIFEST.format(id=name, pid=pid, units=tmp_path / f"{name}.units.jsonl"))
return path
def pred(fold: int, label: int, s: float, *, family: str = "injection", group: str = "", verdict: str = "", pair: str = "p") -> Prediction:
verdict = verdict or ("vulnerable" if s >= 0.5 else "not_vulnerable")
return Prediction(f"outer-{fold}", family, group or f"org/r{fold}", label, s, verdict, pair=pair)
# --- the threshold ---------------------------------------------------------------------------------------------------------
def test_the_wilson_bound_needs_73_clean_flags_at_095() -> None:
assert ex.min_flagged_for_bound(0.95) == 73
assert wilson_lower(72, 72) < 0.95 <= wilson_lower(73, 73)
clean = [pred(0, 1, 0.9) for _ in range(73)] + [pred(0, 0, 0.2) for _ in range(10)]
assert precision_threshold(clean) == 0.9
assert precision_threshold(clean[1:]) is None # 72 clean flags never reach the bound
assert precision_threshold(clean[1:], bound="point", min_flagged=10) == 0.9
assert precision_threshold(clean[:5], bound="point", min_flagged=10) is None
def test_the_lowest_qualifying_threshold_and_unsure_never_blocks() -> None:
rows = [pred(0, 1, 0.9) for _ in range(80)] + [pred(0, 1, 0.4) for _ in range(5)]
assert precision_threshold(rows) == 0.4 # 85 clean flags at 0.4: the lowest threshold that qualifies
assert precision_threshold([*rows, pred(0, 0, 0.6)]) == 0.9 # one false flag at 0.6: 85 of 86 and 80 of 81 do not
unsure = [pred(0, 1, 0.95, verdict="unsure") for _ in range(100)]
assert precision_threshold(unsure) is None
decided, _ = nested_precision_block([*unsure[:3], *(pred(1, 1, 0.95) for _ in range(80))])
assert all(r.point == "DROP" for r in decided if r.verdict == "unsure")
def leaky_threshold(preds: list[Prediction], fold: str) -> float | None:
"""What a rule that peeked at the held-out fold would choose: every fold's rows, the reported fold included."""
return precision_threshold([r for r in preds if r.family == "injection"])
def test_no_threshold_is_chosen_with_outer_fold_labels() -> None:
"""Fold outer-0 holds 80 high-scoring positives; the other folds alone never reach the bound. A rule that used
outer-0's labels would block on outer-0; the nested rule must not, and flipping outer-0's labels must not move
outer-0's threshold."""
held = [pred(0, 1, 0.9, group=f"org/h{i}") for i in range(80)]
train = [pred(f, lab, 0.9 if lab else 0.3, group=f"org/t{f}-{i}") for f in range(1, 5) for i, lab in enumerate((1, 1, 1, 0, 0))]
preds = held + train
assert leaky_threshold(preds, "outer-0") is not None # the test has teeth: the outer labels would qualify
decided, thresholds = nested_precision_block(preds)
assert thresholds["injection"]["outer-0"] is None
assert all(r.point == "DROP" for r in decided if r.fold == "outer-0")
flipped = [Prediction(r.fold, r.family, r.group, 1 - r.label, r.s, r.verdict, pair=r.pair) if r.fold == "outer-0" else r for r in preds]
_, again = nested_precision_block(flipped)
assert again["injection"]["outer-0"] == thresholds["injection"]["outer-0"]
# and the other folds' thresholds do see outer-0 (it is their training data)
assert all(thresholds["injection"][f"outer-{f}"] == 0.9 for f in range(1, 5))
assert all(again["injection"][f"outer-{f}"] is None for f in range(1, 5))
def test_select_on_paired_ignores_unpaired_training_rows() -> None:
unpaired = [pred(1, 1, 0.9, pair="", group=f"org/u{i}") for i in range(80)]
paired = [pred(1, lab, 0.9 if lab else 0.1, group=f"org/p{i}") for i, lab in enumerate((1, 0) * 3)]
held = [pred(0, 1, 0.9), pred(0, 0, 0.1)]
_, everyone = nested_precision_block([*unpaired, *paired, *held])
_, only_paired = nested_precision_block([*unpaired, *paired, *held], eligible=lambda r: bool(r.pair))
assert everyone["injection"]["outer-0"] == 0.9 and only_paired["injection"]["outer-0"] is None
def test_the_calibrated_rule_blocks_only_where_calibration_holds(monkeypatch: pytest.MonkeyPatch) -> None:
rows = [pred(f, 1 if i < 30 else 0, 0.9 if i < 30 else 0.1, group=f"org/g{f}-{i}") for f in range(5) for i in range(40)]
import openultrasast.learn.evaluate as ev
real = ev.check
for holds in (False, True):
monkeypatch.setattr(ev, "check", lambda *a, _h=holds, **k: ev.replace(real(*a, **k), holds=_h))
decided, info = calibrated_block(rows)
assert info["injection"]["calibration_holds"] is holds
blocked = [r for r in decided if r.point == "BLOCK"]
assert bool(blocked) is holds and all(r.label == 1 for r in blocked)
# --- the manifest ------------------------------------------------------------------------------------------------------------
def test_rule_manifests_cost_nothing_and_change_only_the_rule(tmp_path: Path) -> None:
path = write_rule_manifest(tmp_path, "0" * 64)
manifest = ex.load_manifest(path)
assert manifest.kind == "rule" and manifest.arms["B"].rule["bound"] == "wilson" and manifest.budget_total_usd == 0
text = path.read_text()
for bad, why in (
(text.replace("budget_usd: {total: 0}", "budget_usd: {total: 1}"), "budget_usd must be 0"),
(text.replace("A: {label: today, ", "A: {label: today, spec: {sampling: retrieval_ensemble}, "), "BLOCK rule only"),
(text.replace("name: calibrated_block", "name: magic"), "rule.name"),
(text.replace("bound: wilson", "bound: hope"), "bound in"),
(text.replace("primary: paired", "primary: all"), "populations"),
(text.replace("arm: B,", "arm: Z,"), "decision"),
(text.replace("flagged_precision_lb", "auc"), "metric"),
(text.replace("programs: {injection: " + "0" * 64 + ", path: ", "programs: {path: "), "program for every family"),
):
path.write_text(bad)
with pytest.raises(ex.ExperimentError, match=why):
ex.load_manifest(path)
# --- the zero-cost run ------------------------------------------------------------------------------------------------------
def prepared(tmp_path: Path) -> tuple[FileStore, ex.Manifest, list[ex.Unit]]:
store = FileStore(tmp_path / "memory")
store.put_rows(example_rows())
pid = put_program(store, artifact())
manifest = ex.load_manifest(write_rule_manifest(tmp_path, pid))
units = ex.freeze_units(store, manifest, EXAMPLES)
ex.write_units(manifest.units_file, units)
ex.register(store, manifest, commit_of=committed)
return store, manifest, units
def test_the_run_replays_the_cache_excludes_misses_and_proves_zero_spend(tmp_path: Path) -> None:
store, manifest, units = prepared(tmp_path)
chat, caller = paid()
caller.store = store
folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]}
index = {e.id: e for e in EXAMPLES}
cached = units[: len(units) - 3] # the incumbent's evaluation answered all but three units
for u in cached:
e = index[u.unit]
program = Program(ex.arm_spec(manifest, manifest.arms["A"], u.family, store), EXAMPLES, excerpt_text)
program.decide(Candidate(Target.of(e), CODE[e.excerpt_sha], "python"), folds[u.fold], caller, seed=0)
paid_calls = len(chat.calls)
with pytest.raises(ex.ZeroCostViolation): # a paid caller is refused before anything is decided
ex.run_rule(store, manifest, EXAMPLES, caller, caller.client, excerpt_text)
zero, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store)
summary = ex.run_rule(store, manifest, EXAMPLES, zero, meter, excerpt_text)
assert len(chat.calls) == paid_calls and meter.calls == 0 and meter.usd == 0.0 and zero.calls == 0
assert summary.decided == {"score": len(cached)} and summary.skipped == {"already": 0, "replay_miss": 3}
assert summary.spend["score"]["client_calls"] == 0 and summary.spend["score"]["usd"] == 0.0
outcomes = ex.load_outcomes(store, manifest.id)
assert sum(o.verdict == ex.MISS_VERDICT and o.s is None for o in outcomes) == 3
report = ex.analyse_rule(manifest, outcomes, units, resamples=20, provenance={"code_commit": "c" * 40})
assert report["units"]["excluded_replay_miss"] == 3 and report["units"]["evaluated"] == len(cached)
assert report["spend_usd"] == {"score": 0.0} and report["decision"] in ex.VERDICTS
assert set(report["families"]["injection"]) == {"A", "B", "C", "capacity"} and report["min_flagged_for_bound"] == 73
text = json.dumps(report)
assert "org0/repo0" not in text and units[0].unit not in text # counts only
store.put_row(ex.rule_result_row(manifest, "c" * 40, report))
shapes: dict[str, set[str]] = {} # one JSON type per column across the experiment's rows (S3 Select, see test_learn_experiments)
for record in store.rows(repo=ex.experiment_repo(manifest.id)):
for name, value in record.row.items():
if value is not None:
shapes.setdefault(name, set()).add(_json_shape(value))
assert {name: sorted(s) for name, s in shapes.items() if len(s) > 1} == {}
again = ex.run_rule(store, manifest, EXAMPLES, *ex.zero_cost_caller("deepseek-flash", PRICES, store), excerpt_text)
assert again.decided == {"score": 0} and again.skipped["already"] == len(units)
with pytest.raises(ex.ExperimentError, match="zero cost"):
ex.run(store, manifest, EXAMPLES, {}, excerpt_text)
def test_a_call_past_the_meter_voids_the_run() -> None:
caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, None)
with pytest.raises(ex.ZeroCostViolation):
meter.inner.complete(model="deepseek-flash", messages=[], tools=[])
# --- the analysis --------------------------------------------------------------------------------------------------------------
def crafted(tmp_path: Path, *, b_clean: bool) -> tuple[ex.Manifest, list[ex.Outcome], list[ex.Unit]]:
"""Five folds of 20 candidate pairs each: positives at s 0.9; negatives at 0.1, except four per fold at 0.9 unless
``b_clean``."""
manifest = ex.load_manifest(write_rule_manifest(tmp_path, "0" * 64))
units, outcomes = [], []
for f in range(5):
for i in range(40):
label = 1 if i < 20 else 0
high = label == 1 or (not b_clean and i >= 36)
u = ex.Unit(f"u{f}-{i}", f"org/r{f}-{i % 20}", "injection", label, f"outer-{f}", f"p{f}-{i % 20}")
units.append(u)
s = 0.9 if high else 0.1
outcomes.append(ex.Outcome(u.unit, u.group, u.family, label, u.pair, ex.SCORE_ARM, 0, s,
"vulnerable" if high else "not_vulnerable", 0.0)) # fmt: skip
return manifest, outcomes, units
def test_the_verdict_follows_the_registered_rule(tmp_path: Path) -> None:
manifest, outcomes, units = crafted(tmp_path, b_clean=True)
report = ex.analyse_rule(manifest, outcomes, units, resamples=20)
cell = report["families"]["injection"]["B"]["paired"]
assert cell["flagged"] == 100 and cell["flagged_precision"] == 1.0 and cell["coverage"] == 1.0 and cell["meets_target_lb"]
assert report["decision"] == "adopt"
assert report["families"]["path"]["B"]["paired"]["flagged"] == 0 # a family without units flags nothing
manifest, outcomes, units = crafted(tmp_path, b_clean=False)
report = ex.analyse_rule(manifest, outcomes, units, resamples=20)
assert report["families"]["injection"]["B"]["paired"]["flagged"] == 0 and report["decision"] == "inconclusive"
assert report["families"]["injection"]["C"]["paired"]["flagged"] == 0 # 80/96 point precision never reaches 0.95 either
row = ex.rule_result_row(manifest, "c" * 40, report)
assert row["kind"] == "experiment_result" and row["tests"]["injection"]["B"]["flagged"] == 0 and row["looks_taken"] == []
def test_the_registered_exp003_manifest_loads() -> None:
path = ROOT / "plane/experiments/exp-003-precision-bound-blocking.yaml"
if not path.exists():
pytest.skip("exp-003 is not written yet")
manifest = ex.load_manifest(path)
assert manifest.kind == "rule" and manifest.budget_total_usd == 0 and manifest.raw["populations"]["primary"] == "paired"
assert manifest.arms["A"].rule == {"name": "calibrated_block"} and manifest.arms["B"].rule["bound"] == "wilson"
assert len(manifest.families) == 6 and set(manifest.programs) == set(manifest.families)