learn(exp-003): decision-rule experiments and the precision-bound BLOCK arm

kind: rule experiments score once from the response cache behind a
zero-ceiling meter (a miss is excluded and counted, never asked) and apply
each arm's BLOCK rule to the same held-out scores. Arm B chooses a per-family
raw-score threshold on the other outer folds only (lowest t with Wilson 95%
LB >= 0.95); a test fails if a fold's own labels move its threshold.
Manifest and frozen units (353 units, 146 groups, 244 paired) for exp-003.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

plane/experiments/exp-003-precision-bound-blocking.units.jsonl

@@ -0,0 +1,353 @@
+{"family": "deserialization", "fold": "outer-1", "group": "aio-libs/aiohttp", "label": 1, "pair": "ee577ee4b6076fab", "unit": "2523f55d035d1a772a842090bc335e5d755b2fe15b376dd5b6b9e30b9dae82d1"}
+{"family": "deserialization", "fold": "outer-1", "group": "aio-libs/aiohttp", "label": 0, "pair": "ee577ee4b6076fab", "unit": "ca0f6c160d7839204f6fcc1709bcbc749a02a0b3b8bb648e2a238eacbf22ed78"}
+{"family": "access_control", "fold": "outer-0", "group": "alchemycms/alchemy_cms", "label": 1, "pair": "c95b6b01d0c2db2a", "unit": "15540c23be0c42472a49039658ec2ec0bbd8c6b318e35dd040a9d427370e0ef0"}
+{"family": "access_control", "fold": "outer-0", "group": "alchemycms/alchemy_cms", "label": 0, "pair": "c95b6b01d0c2db2a", "unit": "542471980417e964182d7d3db59a05fe12bdd072449f3bd77f7daa5515dbc909"}
+{"family": "access_control", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "bcc636b23dfb5f570d8864b17fe07b6b4fb52b65a40988c32011f1422f74d504"}
+{"family": "access_control", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "dbcf1e67a1be7ccf7d51701f9b8339339164055135443044c23a446b16b3d934"}
+{"family": "injection", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "50408f68114ecb4de3bebd836d52d9afad86b22dd1843641ac8a49b617c09e33"}
+{"family": "injection", "fold": "outer-0", "group": "anxolerd/dvpwa", "label": 1, "pair": "", "unit": "cafb0f00f80dc6beeba40eb95c764bf9a3569193070ba321c28677dfa425ec88"}
+{"family": "output_encoding", "fold": "outer-3", "group": "apache/logging-log4net", "label": 1, "pair": "fe0e104d111c2555", "unit": "088036fab9962be2b96f9162ea4f32190575678d3c5ba96bdfeac178bc22b2c4"}
+{"family": "output_encoding", "fold": "outer-3", "group": "apache/logging-log4net", "label": 0, "pair": "fe0e104d111c2555", "unit": "7fd2b56a816e25358477eee64487fb2c77f34c652703ad0a7120c368492017a3"}
+{"family": "path", "fold": "outer-0", "group": "apache/wicket", "label": 1, "pair": "b6742570f7f0e026", "unit": "7c6c1d27aee6557f1b6bf371b8209003f0d2d1122cfa35b6629eb43f38769cf4"}
+{"family": "path", "fold": "outer-0", "group": "apache/wicket", "label": 0, "pair": "b6742570f7f0e026", "unit": "df8300d84dd784d0be46969ea432ef97b8ce611bae7a896ed4f349bd4d6897db"}
+{"family": "access_control", "fold": "outer-0", "group": "apolloconfig/apollo", "label": 1, "pair": "0f584bb97a154488", "unit": "222a84ffc7c58d45e8f10df221762b23bf20d0287ecd20f375558134928886ca"}
+{"family": "access_control", "fold": "outer-0", "group": "apolloconfig/apollo", "label": 0, "pair": "0f584bb97a154488", "unit": "25e4c9d2941f59f072385a7bde3fd0bd76f9e986ed24b06ce81962ab447f5094"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "archivy/archivy", "label": 0, "pair": "bb3d7baed62e6bb7", "unit": "e8ae88fd2b677e3f64ac94bddb012e3b903afa294ecc87774d10087c0d50489c"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "archivy/archivy", "label": 1, "pair": "bb3d7baed62e6bb7", "unit": "f6b0d241891dc1d897d74d98b6ff7ada2c1f1ce7874ca32097630c06ca53e389"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "axllent/mailpit", "label": 1, "pair": "143b900b99a46cf4", "unit": "200f96cd784ff5c39d8bba22883f22bc8a7c87c6970a7c871802f48a45548337"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "axllent/mailpit", "label": 0, "pair": "143b900b99a46cf4", "unit": "7a0d62ec424849ec4681f4333fbb077e8239054d1e964bd99b40847ad7ccdb01"}
+{"family": "injection", "fold": "outer-2", "group": "browserify/static-eval", "label": 1, "pair": "c12dbc8a50f7fe9b", "unit": "04beb8a0fd054b92c2dd0ed83907330fdee6124d86114df948ac72a297a7d51f"}
+{"family": "injection", "fold": "outer-2", "group": "browserify/static-eval", "label": 0, "pair": "c12dbc8a50f7fe9b", "unit": "c5029016a6fd2e3f9df59dba0ce3c173bbbb0d2b1de22d151086e01e1b272119"}
+{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "0e213aeabacf4205db024ea3c557e720c3aba911424555784917e967d6404867"}
+{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "e8bc65415bd42e43", "unit": "4920230ef773595a9fd9a92a090dff42b48499a1dc5fc8831f941dae55cb2538"}
+{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 1, "pair": "e8bc65415bd42e43", "unit": "49209a810f207eb61382e2ceeab7076936838a5652546f21c187d066e220160b"}
+{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "9e2964884cdf281fa3f18ee3481aa62f9a8a0a195d605da2df5bd71876fe311a"}
+{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 1, "pair": "", "unit": "f9cc6f1bbcd59f5dc0de8f96248f722ea0679f8afa47c53f70832151ce7616d3"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 0, "pair": "e69463f945f9a0f4", "unit": "5feace7b28e260175f80c682e9009025aea6fae3dc75409cabbb955b560d0372"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 1, "pair": "e69463f945f9a0f4", "unit": "799a4f5a5b9324e86c90752f0239f059984f44e8ff1ec0dd518d0d9413e34c66"}
+{"family": "deserialization", "fold": "outer-2", "group": "cesargb/laravel-magiclink", "label": 0, "pair": "768fc9db8a75a351", "unit": "adc6527469618f72b1dd7776814717cbd0056a0c625e05bef74199171ce73f1a"}
+{"family": "deserialization", "fold": "outer-2", "group": "cesargb/laravel-magiclink", "label": 1, "pair": "768fc9db8a75a351", "unit": "da47663a57e370a75cc65b810b8a380aa26d12dcabb1d9b3c02641398ff7d348"}
+{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 0, "pair": "fab4ce020049724b", "unit": "b9be90242be627aa46462143ac85940a906232ebc0b47776b5729475b27d215e"}
+{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 1, "pair": "fab4ce020049724b", "unit": "ece12ea7d87b4a12c4da30faed9eb530b51fc879c271b68ba5304b230cda8e79"}
+{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 1, "pair": "9126ba83905eb8b4", "unit": "d8dc50b7bb40bbcfc3f3d4f1f8d3a4d17eb1c662d5f759efd18656a482dd2e32"}
+{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 0, "pair": "9126ba83905eb8b4", "unit": "f426f9307621b3f2d6a8d1669cdb4c19e962b94e34fbb394d4660cbde5921c88"}
+{"family": "deserialization", "fold": "outer-3", "group": "commenthol/serialize-to-js", "label": 0, "pair": "7ac5e21d9d8f4c47", "unit": "1d1efa2fd7323c264290d4d314c957c853e1c7a9c877a54d6f03ab7d2af2ca86"}
+{"family": "deserialization", "fold": "outer-3", "group": "commenthol/serialize-to-js", "label": 1, "pair": "7ac5e21d9d8f4c47", "unit": "b5dc78c7ac0b76803dc98904142c1139083fb9daafb5bff40e7b5c3f14ee03f5"}
+{"family": "deserialization", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "0894a7212280bab755cd9d93b1f0ede2c51e4fca76b76e5ed9e24ef8f8cf0b9c"}
+{"family": "injection", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "02694b5fb85e9e52bf5eaaab4e0b628f1f44ff3a8d5ad026cc10afdbc4222eaf"}
+{"family": "injection", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "db9a3e888780501dd3c2c125af2f9d47f68ebf02f0e0db8370b2ea09c6f9d070"}
+{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9af96b099dedcc4005e5fcd19ad78f73864e1cfbafbe6953b4edb6260957943d"}
+{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9e7c2a786068a936a6c664cc229d6f39cea1fe57caf0806306f0cfc44515460c"}
+{"family": "deserialization", "fold": "outer-4", "group": "cps-it/mailqueue", "label": 1, "pair": "07d1dc4d299a7700", "unit": "0e664e274dfb438d2f721edc81f8469ac48387dbd5c1b43973cd7e651db46db3"}
+{"family": "deserialization", "fold": "outer-4", "group": "cps-it/mailqueue", "label": 0, "pair": "07d1dc4d299a7700", "unit": "da55f60378c67e91683b9720f2fa0e6659e28c7b392c050278b40713e47c84a3"}
+{"family": "deserialization", "fold": "outer-0", "group": "datapizza-labs/datapizza-ai", "label": 1, "pair": "5a91049be21aa611", "unit": "1ab9936627aa59c3a60ea44074ad44b68aea4f9c04f14c1949dd9c77ca13ff0a"}
+{"family": "deserialization", "fold": "outer-0", "group": "datapizza-labs/datapizza-ai", "label": 0, "pair": "5a91049be21aa611", "unit": "8df72b9eab3f2fd858c40312d0a806cb67c620bd62873826c8732ba8795d9cfa"}
+{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "", "unit": "115d87f37bf1d4ab91fa3010303e81b127f84b817268b60b8fd4b9934efeae4f"}
+{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "2b8101c89c839614", "unit": "7bb7d858a670fc51f81e08c141c2bca3b8b7524254ab679e33b22ce79536e895"}
+{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "c23acf4f38945093", "unit": "82bdc0b6bb0c19e342efa63af0c9014a607461d0682cdd14bd5fea33516fbd50"}
+{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 1, "pair": "2b8101c89c839614", "unit": "886285661b88f14e00ff8e863b31cf7b7afb303ad4aa4aa13b1cffbf1272d5e0"}
+{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 1, "pair": "c23acf4f38945093", "unit": "bf74aae5a48156f183426a80cafca6da219ab13b2ae4b40fa88fca75acb72980"}
+{"family": "access_control", "fold": "outer-2", "group": "delmaredigital/payload-puck", "label": 0, "pair": "d7cc37a3a54315cb", "unit": "4aecd43c6e5b81daf7aadd6a6e22f22c17c08b652a2f5121a49320ca1057b186"}
+{"family": "access_control", "fold": "outer-2", "group": "delmaredigital/payload-puck", "label": 1, "pair": "d7cc37a3a54315cb", "unit": "88123ae860be2fdc28fd6a289cf26c3e0af0689f455cf8c15098e5dc76e5c8a8"}
+{"family": "output_encoding", "fold": "outer-4", "group": "devbridge/jquery-autocomplete", "label": 0, "pair": "81caad96b221c2b8", "unit": "3c2dc720ed159e55629ba737665d93859def9790427289567a513c83c133e236"}
+{"family": "output_encoding", "fold": "outer-4", "group": "devbridge/jquery-autocomplete", "label": 1, "pair": "81caad96b221c2b8", "unit": "e8b9595a320776c8e663fd0b168193c1e64e36a9052784d1219f2c4a91d647f2"}
+{"family": "path", "fold": "outer-3", "group": "dgtlmoon/changedetection.io", "label": 0, "pair": "f9b6599018ef3b60", "unit": "7558d6ee39a311c5459e35fe6f7b12917b7f843b21ab1fbb5e485a07f53b7d78"}
+{"family": "path", "fold": "outer-3", "group": "dgtlmoon/changedetection.io", "label": 1, "pair": "f9b6599018ef3b60", "unit": "b1a6cf450a074788ffca2bebd880435abac2c376e89cb328d7a22d277fdccad8"}
+{"family": "output_encoding", "fold": "outer-0", "group": "dicebear/dicebear", "label": 1, "pair": "eaf2f03db2c502b5", "unit": "38dbfbf745d7cbcc52c6fff83248d84663273fe6a1f1b014eac7776c4bb8f074"}
+{"family": "output_encoding", "fold": "outer-0", "group": "dicebear/dicebear", "label": 0, "pair": "eaf2f03db2c502b5", "unit": "a84d848ef56b72fc15872d88acaae1743c9cdf88095d13b1869abbec6a0d8a8f"}
+{"family": "deserialization", "fold": "outer-1", "group": "docling-project/docling-core", "label": 0, "pair": "7df48bca2c9fabfc", "unit": "7b76ce8f55d61071adb367018cc3e50f04a05d08c7c4919c3806fc1cc8bf41a7"}
+{"family": "deserialization", "fold": "outer-1", "group": "docling-project/docling-core", "label": 1, "pair": "7df48bca2c9fabfc", "unit": "c82ea0adc9c8366aa2b77ce73772038676fc37ff50879e873ff397958fdd7ec1"}
+{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "7f02a09b8d8f2321f2cdade74c2fa1d52442e006149a749a8be768daaa62195d"}
+{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "9b5e439943ce5c2b04d6201b7a843634aa1b13b726813c23450ff54ce03f3dd6"}
+{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "5b836889b4da4921", "unit": "a84c2ce4a4eadf265d4481ad7f6864edf1d7e3084c6ee32cbbfe454bb2946bfe"}
+{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 0, "pair": "5b836889b4da4921", "unit": "caeed7f097a3311fe2e84efd372fe4b4c140f63a80a9f07ba0b61966dbab15e0"}
+{"family": "access_control", "fold": "outer-2", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "c2a2c2d85114a97f892fb1e5500fbd98622457a8c46451e1f9e958ab08a40318"}
+{"family": "injection", "fold": "outer-2", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "29d8be8206c88ba7efd86b56a7dcf8b655e33b4a792c2d18b028c92c6252daab"}
+{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 0, "pair": "f96f0fb245948915", "unit": "0954e6931373915c204b9ec8d2b8ae6297a95667bfd41aadf9035625ad93967f"}
+{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 1, "pair": "f96f0fb245948915", "unit": "5e09cdae2504e834d591b6f92a808427f07550364b5419899f7e6f76084fc377"}
+{"family": "output_encoding", "fold": "outer-4", "group": "ether/etherpad", "label": 0, "pair": "bf11f138fe39675e", "unit": "56f0d0b156992af56141d6368fdb0c5376fc7a44a03991d754f97202b9d77008"}
+{"family": "output_encoding", "fold": "outer-4", "group": "ether/etherpad", "label": 1, "pair": "bf11f138fe39675e", "unit": "8271af5017894167556893da2070188060d06eebc87f27db45c46234c359631d"}
+{"family": "path", "fold": "outer-2", "group": "fastify/fastify-static", "label": 0, "pair": "9d937c2dffb4aadc", "unit": "01ac12bb06e660d0aeef8b1e26dddc5c99d52cc333f7af6d1981eee2818e64fd"}
+{"family": "path", "fold": "outer-2", "group": "fastify/fastify-static", "label": 1, "pair": "9d937c2dffb4aadc", "unit": "9612edf0f927048335e83e9925f01be9aea9b772bdbdc568ad9cc0a17055662a"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 1, "pair": "d4f648048e45f2ea", "unit": "08807c8110b8c1b8635a6883c229fb91f2b9566db8b6c252448a0db7b2ddd47e"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 0, "pair": "d4f648048e45f2ea", "unit": "10869c931053bf77ecaa98a47f56dea27d643d1abed2a4758bcf48a78547ccc9"}
+{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "662e1f36c230d7f1", "unit": "0b73332b25413902c26d5b5dee6767465eed99ff71a814582988ffe0d8ec1a30"}
+{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "8c8e46d0c7d361b784baff9d70f38744b2805ef1ee0aaacaa616bc20a2c6f32d"}
+{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "aca938682c196fd4c85b39b89eeedb97c35f6a5a017651fe9c03c5b727cab7ea"}
+{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "cec00d1509a129364da6ab8537fb3f433fd847e26d073c90311ccdaf7a1b39e7"}
+{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 1, "pair": "662e1f36c230d7f1", "unit": "e0f58a7ac5627de3e7b6cfe5add41d18b0f64b998c1cc788d42eb2cdbbc3fe41"}
+{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 0, "pair": "2c251a645094f389", "unit": "0d41cc634771048f72d07c4aa9feaed669d0bfe3844d61dcdc525a0f899617e2"}
+{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 1, "pair": "2c251a645094f389", "unit": "ca8288d0b8b94dc6b7e4c681936ea610d295fe949cba5f64f09b6de134ef9875"}
+{"family": "access_control", "fold": "outer-2", "group": "forgekeep/nebula-mesh", "label": 1, "pair": "2ac5ec5011615c9b", "unit": "2f427f403af3cb58d776437736c2389805543413fdb91aad98979158c6d00de3"}
+{"family": "access_control", "fold": "outer-2", "group": "forgekeep/nebula-mesh", "label": 0, "pair": "2ac5ec5011615c9b", "unit": "cddc8c5bed0841b80e7b20d0b378e858f8e4ac37fd7b890ce76611d989857f92"}
+{"family": "injection", "fold": "outer-2", "group": "frangoteam/fuxa", "label": 1, "pair": "", "unit": "0c7830c2912ff285855d8a3c8e9b47db4ca4af840baa66d4b5d3f4422f46f95d"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "5383a8c2a78fabd3d17a5b3021125953b5ebbee61f8900c30d016c56b1a3bad6"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "360d8bfd2f116e98", "unit": "7c1989a4ec3622bdaebcf1161de850f22cfed041cb38830ca1ded4fa99d40466"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "ec8c706cc2b20f36", "unit": "848688dcd0fe8d46e6762cc8aee1d82e8fad8a2a044c94b547aea11049c85bfc"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "ec8c706cc2b20f36", "unit": "983cc9231f91baee4e2d626193f3b88aa5b09907f6faab870949bdc9c336b816"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "e4ad86416d5c6668", "unit": "98ff1dff2afe80709b8e61a9c95cfeea12b2110c13b7120478268b4762e2c992"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "360d8bfd2f116e98", "unit": "a1a825d23e34757c8e961e43ae791253e62682e130acdb846b58118588006a98"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "a7fa4207508b3989baabc7a1f0ff2ea5229b366c1cb106e7fc945134b8b106c8"}
+{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "e4ad86416d5c6668", "unit": "ef5c53f3114d0057e694df13dc07cbcb532c3b0d59150eda269b87b021d7a6e8"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "getgrav/grav", "label": 1, "pair": "9ae575048504c690", "unit": "176acf077fd58c3e6f940753179ffa3e8316661f355d94acb5e0345b51925ca4"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "getgrav/grav", "label": 0, "pair": "9ae575048504c690", "unit": "228b34ebc2090b5239c7821d28366ddc663e7118d015a79b45dca6618d5490fb"}
+{"family": "output_encoding", "fold": "outer-0", "group": "gofiber/template", "label": 0, "pair": "a3fdf9ce3441ec86", "unit": "653236018929ab8534c27704a20ec5d358047bc66e451620d2ac32081c91d333"}
+{"family": "output_encoding", "fold": "outer-0", "group": "gofiber/template", "label": 1, "pair": "a3fdf9ce3441ec86", "unit": "b70ca2c43f5929e7b885bd728349e721fd85d7605e2f722a7e8cfe0f0d477b0c"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "groundhoggwp/groundhogg", "label": 1, "pair": "", "unit": "92400e83890882a66ffdea1a405e2354a195442e176d85fb707b0442c44d45ca"}
+{"family": "path", "fold": "outer-1", "group": "gtsteffaniak/filebrowser", "label": 0, "pair": "f9dcf3a0fb0f0a14", "unit": "7ca6dd4f673976b000a657f885af3463740419ce41e14f972af0c5483cb75a7d"}
+{"family": "path", "fold": "outer-1", "group": "gtsteffaniak/filebrowser", "label": 1, "pair": "f9dcf3a0fb0f0a14", "unit": "86c1db3a5e9f843d6546146449171e0378ac61d6c8f44c6dd72fd379cdbb188a"}
+{"family": "injection", "fold": "outer-4", "group": "handsontable/formula-parser", "label": 0, "pair": "6144baeb491f4a56", "unit": "f5c22ce104e9593e5272b6e8b3c6f3b61fa4578552ccea80e6021ee90bdb3ffc"}
+{"family": "injection", "fold": "outer-4", "group": "handsontable/formula-parser", "label": 1, "pair": "6144baeb491f4a56", "unit": "f687e7e95dbe47f602a1e0002a44f94fd48e5b1c33f77ecebc6c2d814fcbf450"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "heartcombo/devise", "label": 0, "pair": "2ef4471c66e5e4ab", "unit": "ad7c8e368fe1699a044e058d5c10b41bd9fc0f91c3f3c384c5dac40af39e71f6"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "heartcombo/devise", "label": 1, "pair": "2ef4471c66e5e4ab", "unit": "d888baf7194bbe7158cb3932f58b47a9a235351d388d10df82a130ddd54c2d46"}
+{"family": "output_encoding", "fold": "outer-1", "group": "honojs/hono", "label": 1, "pair": "5628c54152216646", "unit": "47d081c9f6060218967902f694e61c88f64f57c65bec6fc3073afeb0b77d117c"}
+{"family": "output_encoding", "fold": "outer-1", "group": "honojs/hono", "label": 0, "pair": "5628c54152216646", "unit": "e1297bccf9c3606e0b41d762a69ff5d0a33ae4f7f1506071af6b9cb4a177ff56"}
+{"family": "path", "fold": "outer-2", "group": "hs-web/hsweb-framework", "label": 1, "pair": "e52aa814c36ace1f", "unit": "2d6d74364f1982e91f27f1c995b8699df1fc8f8dce6e1e2cf274bf01b6e9494e"}
+{"family": "path", "fold": "outer-2", "group": "hs-web/hsweb-framework", "label": 0, "pair": "e52aa814c36ace1f", "unit": "4727471fea2cf0a92d43b4b1503f1339a7dde87f872ba8064dacbf855c57705b"}
+{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 0, "pair": "6548e1527e82bd99", "unit": "5542613f6d2eb9421b8213b8bb864894f0ecf9fe64945681019230ef0d75cfaa"}
+{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 0, "pair": "", "unit": "6c2adee34e9e8384535ec1073eda60e2ab528ac77c36747802c303e1f5e9ffe8"}
+{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "7ce70e9b20107cc7ebb87d173da765fc53ffd268e521749a06a9a1f38d746845"}
+{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "6548e1527e82bd99", "unit": "c3e54975464323e58ec96ee4d83e7bc08baf94553d553a8dc3fb28cd9c70ff21"}
+{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "d3173c1e43eb6402a71ea2dec1592913dc078a2194ba1c1d509ab129a3180f6f"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "internlm/lmdeploy", "label": 1, "pair": "0924befc49c7bb18", "unit": "e7e4872db40f0420547ed622fbdd7a072b64a7ba466d1ccfff68e691cee6ad99"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "internlm/lmdeploy", "label": 0, "pair": "0924befc49c7bb18", "unit": "f7d95f43fb2df20ebc4c37fceabdd00ffd8159f404ca384481f1764252382388"}
+{"family": "path", "fold": "outer-1", "group": "jarofghosts/glance", "label": 1, "pair": "772e18821701d60f", "unit": "89a0496c7663bbfd67d660c9f481c12420da2dd1bdb29dc079735ee7a0a49626"}
+{"family": "path", "fold": "outer-1", "group": "jarofghosts/glance", "label": 0, "pair": "772e18821701d60f", "unit": "d3241ceaa50a7d23a4650e6a2a55372b08ec96b35b0bede844bc40e16e818ebc"}
+{"family": "injection", "fold": "outer-0", "group": "jas-/node-libnmap", "label": 1, "pair": "250de1302c7e541d", "unit": "5d03535f2df2acdbc319d9e6b20da65d51f3e810b80f3c8f89b5074a971a3f1c"}
+{"family": "injection", "fold": "outer-0", "group": "jas-/node-libnmap", "label": 0, "pair": "250de1302c7e541d", "unit": "78f977cfc096e34b4eada7f798ad2973ef45728d0d01bfab6ac9e2a3257827ef"}
+{"family": "injection", "fold": "outer-0", "group": "jashkenas/underscore", "label": 1, "pair": "c10bfb006f2da55a", "unit": "1ea4b86dbd84581b1d7b7cabbab9f28c8b213416a60778879666ff42d23fd7f1"}
+{"family": "injection", "fold": "outer-0", "group": "jashkenas/underscore", "label": 0, "pair": "c10bfb006f2da55a", "unit": "7647021f742da24c660b2d407886771786e957076c4cf336def560a570dd8778"}
+{"family": "access_control", "fold": "outer-0", "group": "jellyfin/jellyfin", "label": 1, "pair": "a73421efb106ab0a", "unit": "1f6f207a758582de0c0e6b331b19fe119abe18727d45ccbbe09aabb795f3a292"}
+{"family": "access_control", "fold": "outer-0", "group": "jellyfin/jellyfin", "label": 0, "pair": "a73421efb106ab0a", "unit": "32fbdad39eaf5177db4ad0a11d53e785b30bc7b2376ad9d460f29a08429bd0c7"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "jenkinsci/cas-plugin", "label": 0, "pair": "0458e3fd444409f9", "unit": "b521216bb4106738b3571ad3902d08f6230aabd88cf3a8f9f26f1c8ec01304cb"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "jenkinsci/cas-plugin", "label": 1, "pair": "0458e3fd444409f9", "unit": "c87d2871907798fa2e73d31381e9d63089b0388033f096d247373fec07c2cb36"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "jenkinsci/google-login-plugin", "label": 0, "pair": "6bdccabcf5b9fdc9", "unit": "3d6ddca5eb264d88411b86755909f1d2abddd09c129cabd54c96ff26b4105316"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "jenkinsci/google-login-plugin", "label": 1, "pair": "6bdccabcf5b9fdc9", "unit": "e055b898802279bc63204cc601e29f7f1018efad7228cab0cce00ea5866ecc7b"}
+{"family": "access_control", "fold": "outer-2", "group": "jenkinsci/mcp-server-plugin", "label": 1, "pair": "91bc832e70680c92", "unit": "1060c63ff0fae3af567448f3d4c60d9c159b61ccc358964243eb152a9ac38865"}
+{"family": "access_control", "fold": "outer-2", "group": "jenkinsci/mcp-server-plugin", "label": 0, "pair": "91bc832e70680c92", "unit": "369dffd1e730d7997813adfbebf1481cc68987cae0d0085e47b91a26d5330cf8"}
+{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 1, "pair": "a4144258dffb3b8d", "unit": "7ab301e1614979f44c353e665540582493f78ef4bfc41209562c19b900710f8c"}
+{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 0, "pair": "a4144258dffb3b8d", "unit": "ea95e8387dff32ded9dc7b04c11a76a545b4e2ea6db0076b9b13971a59904bb3"}
+{"family": "deserialization", "fold": "outer-2", "group": "jmespath/jmespath.rb", "label": 1, "pair": "858330e01d3cf9bb", "unit": "033cda2b7cffa5a5fa6abf0b7f2cd27ba78d916eb3cab8c79decccd8776f4a3f"}
+{"family": "deserialization", "fold": "outer-2", "group": "jmespath/jmespath.rb", "label": 0, "pair": "858330e01d3cf9bb", "unit": "d44c4555a5ae19aa17b468ec5cff56b33a0b221df52e04d8abcb656399499dee"}
+{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 1, "pair": "c6a49339d4f0c798", "unit": "dc2e4cd5bd963643702d2b832ae170dd25744162e4076e36687256a09e69d089"}
+{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 0, "pair": "c6a49339d4f0c798", "unit": "f83cd137b85e651ca5299c5a42176b797ed4964cbef31418c96d63c52fdb65f5"}
+{"family": "access_control", "fold": "outer-1", "group": "jsonbored/gittensory", "label": 1, "pair": "5ec37c0f475001ca", "unit": "5d8175fb21a4649bdc8730358e9b87adb90b2aafc4da5fee203829663e4f46a0"}
+{"family": "access_control", "fold": "outer-1", "group": "jsonbored/gittensory", "label": 0, "pair": "5ec37c0f475001ca", "unit": "af845e9e332555096dd350401977b2f07015433f3db8a040891d9470f3f9e355"}
+{"family": "deserialization", "fold": "outer-0", "group": "kevin1024/vcrpy", "label": 1, "pair": "c74525e844650de2", "unit": "2e444ad34ee5f47d73f0cfad0a10c70363337b4ddf9067ecd1b8072d8232652c"}
+{"family": "deserialization", "fold": "outer-0", "group": "kevin1024/vcrpy", "label": 0, "pair": "c74525e844650de2", "unit": "c492d325f5ff82b5f817aa4be1440cec8595054dd0719e07a58ac5501b2ae7eb"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "koajs/koa", "label": 1, "pair": "111a5a80a3abc2c9", "unit": "c7ae403742450deb2bd051ad1231121d2117ba56fa72a038c82d58d207cb3870"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "koajs/koa", "label": 0, "pair": "111a5a80a3abc2c9", "unit": "debe34831afca6f8220b9952036a2d4570babdb5e321a2f8dbaec661a016bd50"}
+{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "9950b85412b021ebabb11eea4c490a660ad380d14c673a87c3fb568ba4bcfac9"}
+{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "656df5ff8e69dcdb1f9ba1ed52d734966eb29773bc745587483a355743b8eb5a"}
+{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "91652a4e776fbce70cde9e4a99c6da5e17da7f8d6d0404867c61eec04161b6cf"}
+{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e53bb3a217dc4737e8dd03671de4f6a895599024070a985a268c65a034295e7c"}
+{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "3d7aba22444811a337abcad0e4fc62990bb704d3a10e72d893342fa94699026b"}
+{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "5f18a3026a51f7140b7aaa3aa8563e8b6fdda6065d95e19e25b1dac4b9ce0e00"}
+{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "0fccc306ce80cb4cbfdcde0a2e56d637bcefa9dd30423d6c54a4f77860afe740"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "1e2eaa45661345f1b6759d968d7696dd0d96d6fba1350a00e4d0a2ebaaa2e391"}
+{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "8b9a8ff0e32a327df75eb8a1811464d5f34690079d5930139dd917063bdd2809"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "245d0e939b7c1ccf098c798492009e3a73ea17400fa613dd865cd369d2e7f383"}
+{"family": "access_control", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "653c48df493f31c2d01c973d1e379a180fbe6a6edc83a500bc62509112175f41"}
+{"family": "path", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "3719fe21d45b50b35458bcd2e263e4baee67adbd3c98adf0eaab6bc6183c5651"}
+{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "1209a6cc8c1548ed83d4a9a09d6a27f706a23e69cf0c261fc5e3a27536a83306"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "362c90495dc468bb732c71f62d71ec308854161de31ad18d2a870056d947ddb6"}
+{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "b52e53fec9707b2b442c4dc7a41d376cf76946d2c45c2eb600c9477fc6baebaa"}
+{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "477e3b90b92e622888dbe0e9d3a23b31271ddfa3104a5ab326eb9e9e765aa01d"}
+{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "660c63c4a73bfe64088ae3249e40d9c5d69a6f8a505ff671217c5708a345c5b0"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "4d60c386c801bedb61c1461fced29a3cfb50b09d6435e8617c60398343719d93"}
+{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "f1fddcc1354a29bef55610ee2b73eaa440192f2f18a0d22d08057b9d1e1b61b8"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "6d9886c38221b2f9aacf83749218ac5ebc354a6fcc4170e6c78bedb92b88296a"}
+{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "5550fec4a81c4c9cb22d7312d8a63e1fb2b03c4aed0c1f316d44c1eb27511bac"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8ff87dd04140f0137ea1891f220b1276156ce10a58be5a49d0c8e07be531ffb2"}
+{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "c6c9bde6630678377d250701073c5c223c7892e6ff8f2f8e0293dea22c5b5e7e"}
+{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "b9b15958973c83f8d7b58372ab5ba58eb191c400125579d5ae8c5b52d5d1368e"}
+{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "b95ec12e4e126ad558e944e2c260116ed6a63f5f3ab97fb5cbcdca75a931ad73"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "38a5545ac9ef67e15db720d1396a2d5915f9d06d6f4772012e5bc95f33770376"}
+{"family": "access_control", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "c8bbe9be91c976be97b7ca387048a7d4537d9f6fcc6cbad0a541d1080629d5a6"}
+{"family": "path", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "ffacd53f1bd75155e63b318fb9cba27ec03e754841e1eb39d1d4fa0b2c3dfca2"}
+{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8fd920175fc43b91b4397acffacb10ab8f7fd0ee38bcf2d2f3ed06a3fad0dfe9"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "86c9598ebe3dcda261a69ae545513f6ce73101f3980066e4506ece99bebea84f"}
+{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e5190fcc57505116f79a31c51aed69f8a1eea13a8ae3e7265f44c72418277ef9"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "2b807ff54e6c47e941124c16d0c27c691cef54f2203060a3a6ce812d478c5261"}
+{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "401aefb4b4791301ace2d770d321c6b26743b0aab7b96f8b410254c704be6d2e"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "22d186c93498e21418b6c1d0547a73374b13041d8afc89a80d9540fa658b6b42"}
+{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "dbfb6df14fe37f8c0efda5f16643297fb0550dfc5ad5805d4e4b844d967773b2"}
+{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "f0da8cfe4ac1e586aa7903d72e67b44500bbd5306d72b4b342d10613efc69222"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "2b8fd73943f456da266434470333a7775f55d9a41953e97a000fdaa3257b6a30"}
+{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "a612091f452e60f28f44eefec641175170dc5db04456a39a099cbb13c50b4d2c"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "4c6600facd83bbf234a32d9e28ac494fb516df6e3d9e9dc28d720ff35252d067"}
+{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "fb0ea8cab8c9430d22d1b0fdd65a69146fa95fbff94e720b4861b96538d992c6"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "e81f91a56058ac1473579767abea57064c53d7d007797a854314814e3c35ee74"}
+{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "5851ecd96a449f98b05f243bfda0a7cf3af8612b27f53c414ae1f4ea67242a7f"}
+{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "328621ef2a94632e5d37ea6267d17b766233384cde35c07e29a4eb672d40ba28"}
+{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "04b2eae2128608dad460ba8275142a41a44e32c4243a29e2a6fb2c9ebf3d8f1a"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "33337002ff5f5a189610b042522ceea254c9d107c1f24771b4f616a0f96f636e"}
+{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "02c09a783cbf058856fc2b448406f3ffa6fbfbacf9b70f4e5e22fd741a5c7699"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "956260164cb21a47c499e5598491031ef303418943f5096f225a87162b588574"}
+{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "8bd8c5aed69eae3019cc2d3654b36bdf2b31435e6a020a389cd103696a2f874c"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "1d8397ef4a9fde7b48e2e1e871bba0f960a802cd7823ecfc10e9dfaddb955ccc"}
+{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "c9db92476032eb4296b950a1eff8fa4aaa26f11542a01a5cf10c7b2e32dc6433"}
+{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "d3e0d0f9b0f4b723080989b5486395cda0a5b186a674022ea70e7fa7ab561323"}
+{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "44ff3b3e61c02744830c4abea872821ca7e0f94950c082740c5c48ef13604e94"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "4103e979a99233c3e71fdfe9f1cdcbab34a2c60b5351b75c5ff5dfc287b962d0"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "kozea/weasyprint", "label": 1, "pair": "6768bfae38c6859d", "unit": "f376519889a00950f5af67eb15c639306ea70da7235d3be4a94dbc19cb096cf5"}
+{"family": "untrusted_destination", "fold": "outer-0", "group": "kozea/weasyprint", "label": 0, "pair": "6768bfae38c6859d", "unit": "f3fea4e334f49ef6fe6b17121fef553eb8e33a9ffa13e6d6cbc4c448da996a7c"}
+{"family": "access_control", "fold": "outer-1", "group": "kubev2v/migration-planner", "label": 1, "pair": "d670253fa209671c", "unit": "1aeebe2e440bdd70525c31f93180f33be93f820db5ba46a2e59c371ef83ee794"}
+{"family": "access_control", "fold": "outer-1", "group": "kubev2v/migration-planner", "label": 0, "pair": "d670253fa209671c", "unit": "b11c8a8a24d3809c0507228568fe7cc05ca0a22ed28cb19019adb13cb619d63f"}
+{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 0, "pair": "33260f3e6c8ea6d0", "unit": "7ca409c7dbc1c38ff005624e212efc6d424cb421e27b482dc8dc0d6881f4c11a"}
+{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 1, "pair": "33260f3e6c8ea6d0", "unit": "fc364d7ed77c7c0508da00e5ba847ec0792f3bc373d8b4297747189ef42d1ae0"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "", "unit": "6cce78d7569ad8db7ca0a61d7078198f666502d26ea6c0afb03173a8a4bb16ef"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "82047c0513e43a83", "unit": "6d2dbed1ab1c9d8ac259dd4227be69641fa59d52e816a8dc547ade01d0b9b0c4"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "98ee23cd6bd31cd3", "unit": "ab4bb39be535d70cd95e9035bb958bfc7418576a6ff40a47551894dd140317bb"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "98ee23cd6bd31cd3", "unit": "b70068cbc646bd01d3e10986e002f648a74e9d798900dc59cf3039b749c95c18"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "ff168d95d8d9496e", "unit": "c6ec9478c125ed780d7013c2807f2cec1878f44253667dfed319c28cbfd17d4d"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "b1bfef46dcd3b9a5", "unit": "cd4d095d8380a2fb7ca02fc6abdbae5249cc06455db2929875533f4aa8b886dc"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "82047c0513e43a83", "unit": "d1bade217be8d345cf5d0f0442ee31924187acbb98779dada95274a77462f696"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "ff168d95d8d9496e", "unit": "d561db250b892b91feed7ae877900e1e25963444f73c7699576e010f06da73cf"}
+{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "b1bfef46dcd3b9a5", "unit": "ed78fd5a7ab30bcd6e8ddd27c706a19cfd89750de4fbf81c0b567946eb94db83"}
+{"family": "output_encoding", "fold": "outer-2", "group": "lepture/mistune", "label": 0, "pair": "25f75d77a669b54e", "unit": "0f2825568461562f2a915a4f16ef8a4931290fd20a9801470aeb12677e66e3f1"}
+{"family": "output_encoding", "fold": "outer-2", "group": "lepture/mistune", "label": 1, "pair": "25f75d77a669b54e", "unit": "bf22bb44bcff4a5baf5d550c9d3e83e5f3ca607475fb51deaf4fb6fccff3d73a"}
+{"family": "output_encoding", "fold": "outer-4", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "cd80750447cf7af67ad809279718d98106c71705fef9ff7254728bfff4ece5b1"}
+{"family": "path", "fold": "outer-4", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "2ce605ab5ce7ad3a1fb983cd6bddf3924e5da3010d2252a0a4640aab4e5921a0"}
+{"family": "deserialization", "fold": "outer-1", "group": "litedb-org/litedb", "label": 1, "pair": "b0a899116dc36b85", "unit": "510ad48af03ba4a1a2a06dd254b135311dddb62d9fe5a2e8e8b12627586e6b13"}
+{"family": "deserialization", "fold": "outer-1", "group": "litedb-org/litedb", "label": 0, "pair": "b0a899116dc36b85", "unit": "b414718a8a5f79739b88db24da6ad9da8e48bcb6b5e2dc99158848cdaa226767"}
+{"family": "deserialization", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "8e22ee30441b20484090659b0c97423c711f5d10219041bf3a399623fc056a07"}
+{"family": "injection", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "0f9070df9dc334f981cb01008c260b8bff78dbdeb48ce672830ed95caa8406fb"}
+{"family": "injection", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "5e352f797b1422b0df477bce4730a9ce33cbd0837c3d63da94e4d8801a9a4b64"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 1, "pair": "dadccb128ac0e120", "unit": "505097494da5939404ad7d5b4af2b1c4a10efe94048084492d6ce60ebaa04c97"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "dadccb128ac0e120", "unit": "84ff8ca2d98a1b5d1ca4bfaabe7d19e03a396042b228c8d18d1185a178389702"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "91e261c54bd2c3c3a9ac04e48894ecabaca5b13e7db4fbd5578f9d266098117a"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "a308dfd59bc398824e2454bf6cf6cfd5b7880e5b948a82424f14753d2b6fcdbe"}
+{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "ba24eb554458bc97314ed561a7b2f4a4e643dde1205c32de4c8fcdd8c46e575b"}
+{"family": "output_encoding", "fold": "outer-0", "group": "mganss/htmlsanitizer", "label": 0, "pair": "24d8f1840b67fbe5", "unit": "7ffa4a229a62266788fbf64890f81f20358b3eb8c35d3ab87d0c339d5bb4b507"}
+{"family": "output_encoding", "fold": "outer-0", "group": "mganss/htmlsanitizer", "label": 1, "pair": "24d8f1840b67fbe5", "unit": "d19493ed7941e1cd25a76dc50504d97398770d07ebc302dd66d83441e946c468"}
+{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 0, "pair": "9f8a60eb48d74bc8", "unit": "80fce34f2377731e3de4355c7ea33bb20fc2fc00a5f388565b23018c7a3ed008"}
+{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 1, "pair": "9f8a60eb48d74bc8", "unit": "b68e62722a327c9d5ce6774eb96527451729410768efc0b5c7ba604ec468052d"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "miniflux/v2", "label": 0, "pair": "997f9d2efc32302d", "unit": "b587bb499945a9f5f0749eb4d57cc1027cfd21c8292bd32be4d4e1cc1b1db261"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "miniflux/v2", "label": 1, "pair": "997f9d2efc32302d", "unit": "f8d9c9a7c12797b3caf042e6bfdb89a5072c4eae1fd36be6196f9fa326cdf257"}
+{"family": "output_encoding", "fold": "outer-4", "group": "mixxorz/slippers", "label": 0, "pair": "8e6b4b4e28981590", "unit": "3cce3e8f9ff3f1d261f9b96c844c1e4da07bae72d5393897b3a929be25372edc"}
+{"family": "output_encoding", "fold": "outer-4", "group": "mixxorz/slippers", "label": 1, "pair": "8e6b4b4e28981590", "unit": "b105689ac24ed334294c434a77bb6d56d5d598830ffe8a5d60ab3be5290b47ad"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "0a36cd5177f3b2c20b433db6bfffe5ec5b7e74cacc34a86ac581f7a0a559432a"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "e7fc97d83013c9ff", "unit": "5ab444e64297fc19065478104c7ebdabe23cc415f22adafd0cf82c98e4c5cae3"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 0, "pair": "e7fc97d83013c9ff", "unit": "6b462ea30c1afda572600a10324249223932d36a032eacafac8c837bccbdce87"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "84d6a4202a082e0d", "unit": "8175082f40d8d61d44b54579c84a808e25b89110be5d8fd82feedf7dd5e0d5ab"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "8abefca43ee4ae2c797033da31f85c17bfa7be6b35413d70b0216415c5db9a09"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "91d213946859f33d3f243c43aa8285cc6e8ecb3db1de938f32af775a70607f8e"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "a244d9f010ddb43a09e82363dfabb0e406d411fb5de08ab7d6c9ee367f2a15c3"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "d27d174d6e8e5f1682a1c42723705e89f0d18c95ecedd0567e313daa9a0aae44"}
+{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 0, "pair": "84d6a4202a082e0d", "unit": "f45608ed47fa11ca25093641f6021ec06076c18676abac9b809f31b3b7713de8"}
+{"family": "access_control", "fold": "outer-4", "group": "moquette-io/moquette", "label": 0, "pair": "55d1ce4f29dd1583", "unit": "a877ce3462da316bcdd82ce9db1d9887e887eaa83157922edb1663413edae1d6"}
+{"family": "access_control", "fold": "outer-4", "group": "moquette-io/moquette", "label": 1, "pair": "55d1ce4f29dd1583", "unit": "aee67be0ba31f99c7a24faa1bad6178c6d37361a308cf5aa10138e6ce4bbcfeb"}
+{"family": "path", "fold": "outer-0", "group": "naorsabag/openhop", "label": 1, "pair": "16e529fe8c5486b2", "unit": "1d21acd0171726775d3da4589b660645fcd43cc026042a2b27e0e0e8374eceb7"}
+{"family": "path", "fold": "outer-0", "group": "naorsabag/openhop", "label": 0, "pair": "16e529fe8c5486b2", "unit": "f8e004b67ac98dc3392ea4115051e6bd10ce22eb638a762f0f16e1d4200bf129"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "nuxt-modules/og-image", "label": 1, "pair": "ef603436079b0005", "unit": "77f843b5044bb5da935df4b2620bf2abe9e7dceb3147439e883927112eaec22f"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "nuxt-modules/og-image", "label": 0, "pair": "ef603436079b0005", "unit": "f2ce95f0e44e73986784b63a702ffa4035c423056cac3667a63140a7e8d795c2"}
+{"family": "deserialization", "fold": "outer-3", "group": "opensearch-project/opensearch-ruby", "label": 1, "pair": "1945f28a3fc5bd55", "unit": "64c0b7ccad09192eeaf590befe0de0087d5b48a65a4f6cef41415f3f54a8f536"}
+{"family": "deserialization", "fold": "outer-3", "group": "opensearch-project/opensearch-ruby", "label": 0, "pair": "1945f28a3fc5bd55", "unit": "b55cd6759d4b7861e589052aa534bd0554586db46a329ed3beed7244d579e8c4"}
+{"family": "injection", "fold": "outer-3", "group": "openssl/openssl", "label": 0, "pair": "1c9e47835232f644", "unit": "90e220d3540b88dd9efc8e66c8ab519d4cc78805f16bed585b795c78cfbb4d58"}
+{"family": "injection", "fold": "outer-3", "group": "openssl/openssl", "label": 1, "pair": "1c9e47835232f644", "unit": "b5a185b05235f70f6cf27e2adba919f4d8393dbdf2d99de095548a21946aeaf3"}
+{"family": "access_control", "fold": "outer-3", "group": "openwisp/openwisp-ipam", "label": 1, "pair": "9b2b34cf62a76ea3", "unit": "91a3984e47f17512db8523e04f863525c67db18631d8f11776cf61ca7221c91b"}
+{"family": "access_control", "fold": "outer-3", "group": "openwisp/openwisp-ipam", "label": 0, "pair": "9b2b34cf62a76ea3", "unit": "d91f1e14282a0c3c8da57c6935a1625e881374bd205b5baa9e4a341d9109ae21"}
+{"family": "access_control", "fold": "outer-3", "group": "oqtane/oqtane.framework", "label": 0, "pair": "a20cdf9dd208720d", "unit": "18a26df68e4b81d411cfbcc8155a7e6f382753d6e1ad95eece7ed24fa8c44324"}
+{"family": "access_control", "fold": "outer-3", "group": "oqtane/oqtane.framework", "label": 1, "pair": "a20cdf9dd208720d", "unit": "e6fb74441fb585d7ef5855317fba795ef0f0a0e6eae9970a49556ce3070c3254"}
+{"family": "path", "fold": "outer-4", "group": "owen2345/camaleon-cms", "label": 1, "pair": "d6930fe7e752b7ec", "unit": "06f0341dfbf1aa9504083737df5d9be3cedfd4e3a198799d8075a29990776800"}
+{"family": "path", "fold": "outer-4", "group": "owen2345/camaleon-cms", "label": 0, "pair": "d6930fe7e752b7ec", "unit": "30e2cd8bd381d7568b78eaf982f562d278a6a8d942d5282466532c1441bdf0ad"}
+{"family": "access_control", "fold": "outer-4", "group": "perses/perses", "label": 0, "pair": "2d482951d7380c4f", "unit": "5c1a721da6f85d1d5dfc8cbe5e9809b093a09c059e5abeebed2cbbee55fbd498"}
+{"family": "access_control", "fold": "outer-4", "group": "perses/perses", "label": 1, "pair": "2d482951d7380c4f", "unit": "6cabd1c4b5aa94cafdd07c0c6a3f9abfdc03761ddc5b0a2bebc72a508d020b8b"}
+{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "", "unit": "2300c097374b9098671dcbcd42d02d08172e0f3064a8323223d0fe79a6106aab"}
+{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 1, "pair": "0664148b0b192456", "unit": "d702617aaf4d19811baaa18386a3c8b0eb35cd3fd72e8d90611d34850e990858"}
+{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "0664148b0b192456", "unit": "f5ed13d811c5b16d46cbd4a3b9edb8f7d2db49acbd49d95f39f33eaab38f58f5"}
+{"family": "access_control", "fold": "outer-1", "group": "publify/publify", "label": 0, "pair": "1972ea9f72b178b6", "unit": "3ddabb356d78e7c65ec8f88b6da8e4fcabc055ae73f2f6d92c50efddc5d4fcbe"}
+{"family": "access_control", "fold": "outer-1", "group": "publify/publify", "label": 1, "pair": "1972ea9f72b178b6", "unit": "6875dd5e9c0ff8a1b693d29a109cb2e62e6ff4081df40868aca70e4770476757"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "04aa777130c02a06", "unit": "16aa43c87956c44a4b115245140eefed704095d37cec0cc6410377403cd2fa7a"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "23744137e0485a36f686f5f1b5666780fbed3ca522e8c6d35687a391ef504498"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "67565d99fe28c68f", "unit": "2692316792e6f02fb51578cf30d71bba6c73bf5420c36f709681c8eaa69807d4"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "2e36f73002c416a68d077bffb0335d82346c7507f2478353aeac005fce5a9cbc"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "04aa777130c02a06", "unit": "67721f67055eb6fed955ff22bfe4060344662eccb369431835a2116ad120c4f8"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "8fc1c96fa6faa4d3966ecc42876189b18ddab779d58f47cdf28a1155f48a6ac4"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "67565d99fe28c68f", "unit": "ab535098a8e709b022afee1da15dd1fc371d389056938ee4fa4971b2966c823a"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "d46f94235bc462360097e7a22e1d442d95f049306631489a4d8b25f647a95bbe"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "edd6dd4364442f66384d14e9317ef2f6234e32ab78327a45741b8a816b4332ee"}
+{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "fc7df6887ac719d4615934c0e8c3296728c2f6089f233158c59989027c802d72"}
+{"family": "path", "fold": "outer-2", "group": "rack/rack", "label": 1, "pair": "c0412509f15fe3e1", "unit": "63bda2a71ad7155ad2f59aa9086efef3fd3710dc82c57f4b32ec8bf561460239"}
+{"family": "path", "fold": "outer-2", "group": "rack/rack", "label": 0, "pair": "c0412509f15fe3e1", "unit": "b1c9a070c93764655786c3de5fde536c150e319efe47548614cc544b73a8ee2b"}
+{"family": "deserialization", "fold": "outer-3", "group": "rails/kredis", "label": 1, "pair": "feed5f7de11289a4", "unit": "71531567b2b8609ae0d31cb177b8faea4f5fc61923e513b0154a2c941287ce2b"}
+{"family": "deserialization", "fold": "outer-3", "group": "rails/kredis", "label": 0, "pair": "feed5f7de11289a4", "unit": "d6be6d397a301a07faa761163d3d9e270d71d23d3bdfd084eeafb1e62b91dd71"}
+{"family": "deserialization", "fold": "outer-2", "group": "redisson/redisson", "label": 1, "pair": "ffd4b04a7bebfaf8", "unit": "54f7519d3d59b6d8d69a5f8f7bd78e360d3fabaae00b7587c37e388719169370"}
+{"family": "deserialization", "fold": "outer-2", "group": "redisson/redisson", "label": 0, "pair": "ffd4b04a7bebfaf8", "unit": "c5e42c9ab2ee54e46700ab0b64e239b567fbc75ccaceaf6cbf20687813663332"}
+{"family": "deserialization", "fold": "outer-0", "group": "remix-run/react-router", "label": 1, "pair": "73f2a014c6a699ed", "unit": "44ba9d9d8e75769c1444ebf37a64941a845b4bcae678ec79d64dc84e5cc067a0"}
+{"family": "deserialization", "fold": "outer-0", "group": "remix-run/react-router", "label": 0, "pair": "73f2a014c6a699ed", "unit": "b430d488c486cef26b1c661f773e7e9e4a177eeea73537a033eec98d532e4eed"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "remram44/taguette", "label": 1, "pair": "cb0c638282db0d37", "unit": "a49e6ad2e7c2a5ad0d68d5d69c263306b501fc0dc9382e701fb5fe680611196b"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "remram44/taguette", "label": 0, "pair": "cb0c638282db0d37", "unit": "dee28099d3b37cbd0ee14412122ecc5f8488ed6bec01863520b87706a040fba1"}
+{"family": "injection", "fold": "outer-4", "group": "roest01/node-pdf-image", "label": 0, "pair": "09bce271e8530fbc", "unit": "35eb19b4c6176b37287e2123bb98ec4045ce6bde7cc3864dde317c024b0935d8"}
+{"family": "injection", "fold": "outer-4", "group": "roest01/node-pdf-image", "label": 1, "pair": "09bce271e8530fbc", "unit": "dd33291cc494bbc77d44e9b13d16f171d32c9746637506f315f375e8a9ddee95"}
+{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 1, "pair": "40d8d81a764c508f", "unit": "7deeed73c29c74ccbed29be049e109d363d41e4b5eea245ee57780b85f0e137a"}
+{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 0, "pair": "40d8d81a764c508f", "unit": "b5cdadf848e67910550666ba2a6ac67b24112e501c02dba1865766a316317a23"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "", "unit": "15dbea384b3255457a44d9382efc6944dd27f1d411d903344a5d5e5fd194b72f"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "b2ac284bd70dd684", "unit": "225890ba6694dc8f4fbab46e9c851dff8c095e906cf14db15d01ebce0891aaab"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "b6eca35de93a5be8", "unit": "42eb8427415704e060e73b5ce0943b014ddf380ca756d5ec47c558e59a79895a"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "b6eca35de93a5be8", "unit": "4c9cdcc11a18641ef41e9308712c5e8b830ec9ad5ebc3a956c8701e4f77b39c0"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "6a53939838afbdd7", "unit": "5794809e595311c86909d673a570d003aa025e4dcd6abc5641b641d935beb0b1"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "46d3ec7d49cb8ca7", "unit": "5f7a63ec650915962ee5255f5416df63ab882d6107f3b540be36d481798a8944"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "6a53939838afbdd7", "unit": "97a7018535ab511622e3ade9e53d570ffb1c80633eaa372fb76e749b3a586c8f"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "46d3ec7d49cb8ca7", "unit": "a84e9ab47f4f2b584684dbb81575758d9032fe3f2ed53bef478cb9dcfedaf677"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "b2ac284bd70dd684", "unit": "aeceb1ebc863ebbc3b69ae56c5b78d2dba005132b75cbd6af929bb0bfe25da9b"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "f42cab375ace1739", "unit": "e329d4a6ac757680bf3d292a81f1f74f8a8a9b482d1651efe9f503f6a4de8796"}
+{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "f42cab375ace1739", "unit": "e73c5aac41c0b93072223a3097fb175548ce893e0678ff3a22bc7571745a17e0"}
+{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 1, "pair": "", "unit": "52354ccdbc16f209e4692d97d5237e04ed030cf4a5e71f77b5a767962b355b4a"}
+{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 1, "pair": "7d23be667536afff", "unit": "b5016a5a53409491ba805ed8718599b84250b7b6914da0434b0882b0b3578dc8"}
+{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 0, "pair": "7d23be667536afff", "unit": "f1b01102968568d3ba1b2fdc70f4ae281f433fb20d29eff2a55fc45555ddbfcd"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 0, "pair": "1199de75f426940b", "unit": "10fdc236f43a02b7e291eb8bf6ada297a2ae98086a63b171057bc7bfc37a3377"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 0, "pair": "22112c31603bbe9f", "unit": "5eb3b7523b5a196dcaacba08d68457f9a45e31503ea02745ca3a7d0542f9881c"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 1, "pair": "22112c31603bbe9f", "unit": "6bd5bdfd81b0a3374d59cfb2c1f7fa2b27887f4672f4485e77eb5234b569983f"}
+{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 1, "pair": "1199de75f426940b", "unit": "c09dffd04708540845c8d1dd77a2bd989bd35f264abf652a2db55f8cb1faf7a9"}
+{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 0, "pair": "7aab025ea7b97019", "unit": "7487a34cdfdb532a5096764a47a3e565fa509605c1df0e2b105eb9c751fc5307"}
+{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 1, "pair": "7aab025ea7b97019", "unit": "9c0be2a58741f095fd0a9cdc60d4ca4039de1529699d5701dc4d0bcf7d5bf90c"}
+{"family": "path", "fold": "outer-0", "group": "simonh1000/angular-http-server", "label": 0, "pair": "4de5441f1776f083", "unit": "a3a35d573da308cd1d0614a26616bbd37b360d9ea416894cdeffdb59abd691d4"}
+{"family": "path", "fold": "outer-0", "group": "simonh1000/angular-http-server", "label": 1, "pair": "4de5441f1776f083", "unit": "dc33fc26c1bf7c67e5439ddfdad06c0faf16c796e0fe8a46b6e6a541e708676d"}
+{"family": "access_control", "fold": "outer-3", "group": "smithyhq/sqladmin", "label": 1, "pair": "d76d5232a8587127", "unit": "1ec32d427822d1ddd3b0d7245f5771ddeb56c146ac8ff9959dd85760ff659a20"}
+{"family": "access_control", "fold": "outer-3", "group": "smithyhq/sqladmin", "label": 0, "pair": "d76d5232a8587127", "unit": "5143b19a1a1e6b9162b80594ed7fb5c0c57ed0b8abfd793bfc71089552d57638"}
+{"family": "injection", "fold": "outer-1", "group": "soyuka/pidusage", "label": 0, "pair": "5f6e814444b4acd8", "unit": "2e8aa07b1aa4bfc5349b0d4464f3f2a6500aa55ff765b0139e9901ed34151ffa"}
+{"family": "injection", "fold": "outer-1", "group": "soyuka/pidusage", "label": 1, "pair": "5f6e814444b4acd8", "unit": "e26c726c08be761b22fcc6ec17fc1fa7aba4769f08957f024e5545906df62797"}
+{"family": "output_encoding", "fold": "outer-2", "group": "spatie/schema-org", "label": 1, "pair": "888e955fb6cfb4fd", "unit": "6f1a431ca97afc34ceec48393eb134a8928a312e6f761eb907c6b5855cac6208"}
+{"family": "output_encoding", "fold": "outer-2", "group": "spatie/schema-org", "label": 0, "pair": "888e955fb6cfb4fd", "unit": "7b62415d17569924d16238a4aa28ab1af2db6561f56be9f59d8075fd1bc4a941"}
+{"family": "injection", "fold": "outer-3", "group": "strangerstudios/paid-memberships-pro", "label": 1, "pair": "", "unit": "ac905d12b92954263c046bd0138a682cea86cf8476336d6baa664ade28d34ac6"}
+{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 1, "pair": "218a45d869bd7e39", "unit": "309ebac468b7ab054c8396ed495770385ae3590caba43709745ba2d1db4b6d54"}
+{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 1, "pair": "", "unit": "3cc8613813dec13cb1b7b45b296688da883f0b36cd2c4fc1da6e81703fb59ab9"}
+{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "218a45d869bd7e39", "unit": "9bd2331dbc260451355307e4c37bae17ad6676654c66181934e1b35fb174cc5a"}
+{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "b087ac4d0989e0b72698a9644ef4eac8073b4a15bb21730d63e4730b5cc58a60"}
+{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "c5d1e8d19560c997018fdca12eb60f1799be76c33be058b88472f5a2fc6709be"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "22befc17a75a688e2f9358d2ddf9a7b197da44cd95dbdeae5e0191b113ee54d3"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "63f496d9ff53e39b", "unit": "6a98f8145776c78793b37156e9a81757841bb6de80016ab4fa575209d724db3f"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 1, "pair": "63f496d9ff53e39b", "unit": "7ac60c96c15f8517e089cc2240aedcf1fc30e54a41e35fe03c6b965bc393fbd6"}
+{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "ae39cb55ef2a87c5347e0ddc40975f142450e82de4909e93ba62a7ed0efb4134"}
+{"family": "injection", "fold": "outer-4", "group": "thenables/thenify", "label": 1, "pair": "d1b19898f8deaa41", "unit": "3763c7b407ad471820b5f8fa25bde0ba6d91e15c07c56681de4b41add418b0b1"}
+{"family": "injection", "fold": "outer-4", "group": "thenables/thenify", "label": 0, "pair": "d1b19898f8deaa41", "unit": "bc8b934938c30687f3d4f982392d2cfa9504a916b463ccb5c46fe8a30eede488"}
+{"family": "path", "fold": "outer-1", "group": "thumbor/thumbor", "label": 1, "pair": "f5f33fa6fe7b8492", "unit": "938bbf146a9037e687190bcee82c047492dffbe90efea40a49a647f81a0f1ed0"}
+{"family": "path", "fold": "outer-1", "group": "thumbor/thumbor", "label": 0, "pair": "f5f33fa6fe7b8492", "unit": "b90c9f7e0011685f1a8e853a9ed0ae37fd437ef6d6eb0ba17c240aec2c22b603"}
+{"family": "deserialization", "fold": "outer-3", "group": "tomasnorre/crawler", "label": 0, "pair": "f74364dc8e35c8ba", "unit": "11e760e99c5c852676c9288e18a250febb3854e9047b51e8b9ecc9f3b423b6f5"}
+{"family": "deserialization", "fold": "outer-3", "group": "tomasnorre/crawler", "label": 1, "pair": "f74364dc8e35c8ba", "unit": "5fc980194040f4c4d803075d4085618cc0260ae8f3454b26b9b704d30200ffea"}
+{"family": "output_encoding", "fold": "outer-3", "group": "tornadoweb/tornado", "label": 0, "pair": "db1c51833994ec55", "unit": "c2ce8320c778dd55434d7413844797b68e6fcabc4dc509458a9d60ca70b88c60"}
+{"family": "output_encoding", "fold": "outer-3", "group": "tornadoweb/tornado", "label": 1, "pair": "db1c51833994ec55", "unit": "f2d6843547af1276ae8a4dff3a7786913fa79dc69540d4fbb95c4c091e27c7f2"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "truelayer/truelayer-dotnet", "label": 1, "pair": "f4690679bf9d80aa", "unit": "1a4e24df8d08672ecde4709a77ea5c437b2198fecff8bfff373b63c9eaed6e63"}
+{"family": "untrusted_destination", "fold": "outer-1", "group": "truelayer/truelayer-dotnet", "label": 0, "pair": "f4690679bf9d80aa", "unit": "78711973ec84867671314dd7878769092b32f59860eafed50a0f6979daee76fc"}
+{"family": "injection", "fold": "outer-2", "group": "ultimatemember/ultimatemember", "label": 0, "pair": "85c15e3fc2b370b2", "unit": "0990d8f41cb07c1cf217812e51a7025381de4ba68d1be3799e26241d81dcea67"}
+{"family": "injection", "fold": "outer-2", "group": "ultimatemember/ultimatemember", "label": 1, "pair": "85c15e3fc2b370b2", "unit": "9da1d63f0c4da270f66eb87bf9c342ace883a0aef78c4643e254159b5da8fce3"}
+{"family": "output_encoding", "fold": "outer-0", "group": "viewcomponent/view_component", "label": 0, "pair": "8f5a6abedd742bbe", "unit": "71e3ced643a244100cc3fde234f5b4b27655214c7ae0ff67ab61e3e48a8fdf78"}
+{"family": "output_encoding", "fold": "outer-0", "group": "viewcomponent/view_component", "label": 1, "pair": "8f5a6abedd742bbe", "unit": "7c8cbfae87127042f9f43621345b4f7958bde27a3ff9db76254063e80735f3ce"}
+{"family": "injection", "fold": "outer-1", "group": "vincentmorneau/apex-publish-static-files", "label": 1, "pair": "c9538342f2d7efca", "unit": "2b8db57d65ef1d67c457ff5a336e8a542d03387395d69d5e16815c4f00f40723"}
+{"family": "injection", "fold": "outer-1", "group": "vincentmorneau/apex-publish-static-files", "label": 0, "pair": "c9538342f2d7efca", "unit": "e2ed113dc21ad5f7caa71ba4a5ece8f8f900a97fd0d00dc85f7ac79ff5bed19a"}
+{"family": "injection", "fold": "outer-1", "group": "vvo/node-whereis", "label": 1, "pair": "8acd426c5771b37c", "unit": "983e146a60e061254b15dc8dd573cbba57f8dd0358f23ea1b9c313ef69069af5"}
+{"family": "injection", "fold": "outer-1", "group": "vvo/node-whereis", "label": 0, "pair": "8acd426c5771b37c", "unit": "f80dfec45231e4778078cb10f00128ed445177cac0a062fc2df5ced5c0171f21"}
+{"family": "deserialization", "fold": "outer-4", "group": "waterfutures/epyt-flow", "label": 0, "pair": "9b84800ce5c395cb", "unit": "9b7dfdceae03fd86ad3260a03f85bcdcff8f32f10c265d23743ff8dc98eeefb8"}
+{"family": "deserialization", "fold": "outer-4", "group": "waterfutures/epyt-flow", "label": 1, "pair": "9b84800ce5c395cb", "unit": "b99dda150c0da3defa5f4c527ef6c402f5589dfb2f2f03f3e5e424910eb48c53"}
+{"family": "path", "fold": "outer-4", "group": "webpack/webpack-dev-middleware", "label": 1, "pair": "ad64e8fac4ac3ee0", "unit": "36dfd01915a553d86576e8ecd3b737b8e2b297aa542f58cc83b448060e74398a"}
+{"family": "path", "fold": "outer-4", "group": "webpack/webpack-dev-middleware", "label": 0, "pair": "ad64e8fac4ac3ee0", "unit": "c2b6d40f130d35bbfc16fab9411c664fa06321c59fb1d901a1987a69cb09c72e"}
+{"family": "access_control", "fold": "outer-4", "group": "wintercms/winter", "label": 0, "pair": "d48c08b19f560848", "unit": "03d02c9e25eec0f380b125279fce35e8867065bf6e1ec75f9f39729ce6efdc77"}
+{"family": "access_control", "fold": "outer-4", "group": "wintercms/winter", "label": 1, "pair": "d48c08b19f560848", "unit": "abbea69b7d0fa68c3ace442c25597e128cde354b417787b5913fd10d666c0d19"}
+{"family": "output_encoding", "fold": "outer-3", "group": "xyproto/algernon", "label": 0, "pair": "3844ed3d703d7ef4", "unit": "445efda074e4d9f6c1634225eb08a3c3eb1257311c995ce891df2c791a6f5c8b"}
+{"family": "output_encoding", "fold": "outer-3", "group": "xyproto/algernon", "label": 1, "pair": "3844ed3d703d7ef4", "unit": "8ac27b08549353ada819caaf99a5180d2f2f515b03aa8cd99a5d37b527ea3a5c"}
+{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 1, "pair": "a63eae7d621a985e", "unit": "04ca0b583b09c3b4f67bd2496a4607ecb134d24b7bdaf8960fa151e40ebcc447"}
+{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 0, "pair": "a63eae7d621a985e", "unit": "ecfa2a67726ca9d3e2621a4d8be60145095b76aa18ecf0268212341d63752871"}
+{"family": "deserialization", "fold": "outer-4", "group": "zumba/json-serializer", "label": 0, "pair": "0d9f529b7f0084b6", "unit": "35aa369236f510634b3aa7b54b04e9da0f80877e515786a888e760f8fe33c524"}
+{"family": "deserialization", "fold": "outer-4", "group": "zumba/json-serializer", "label": 1, "pair": "0d9f529b7f0084b6", "unit": "ef4f8613c0914eb815504e00cd6ee05df38352fd3b27826798e6623594017042"}

plane/experiments/exp-003-precision-bound-blocking.yaml

@@ -0,0 +1,65 @@
+# Experiment exp-003 (learned-decision-engine Req 6, Req 7.3/7.4): the BLOCK rule. Registered before it runs;
+# `ousast learn experiment register` records this file's sha256, and a changed file is a new experiment.
+# kind: rule -- both arms decide from the SAME scores: one scoring pass of the six slice-2 incumbent programs, replayed
+# from the response cache only (zero cost by construction: a zero-ceiling meter around a refusing client; a unit with any
+# uncached response is excluded and counted, never asked; the meter's calls and usd must read 0 or the run is void).
+#
+# Threshold selection (arm B), fixed here: per family and per outer fold j, the candidate thresholds are the raw scores
+# s of the training rows, i.e. the family's paired units in the OTHER four outer folds (each of those predictions is
+# itself out of repository: retrieval and demonstrations never cross its own fold's boundary). The chosen threshold is
+# the lowest t such that, among those training rows with s >= t whose majority verdict is not `unsure`, the Wilson 95%
+# lower bound of precision is >= 0.95. No such t -> fold j gets no BLOCK. Fold j's units are then flagged when not
+# `unsure` and s >= t. Fold j's labels never enter its own threshold (tests/test_learn_rules.py).
+#
+# Pre-registered expectation (power): a Wilson lower bound of 0.95 needs at least 73 flagged units with no false flag.
+# The frozen units (353 in 146 repository groups, 244 paired) hold 38 paired positives for injection and 16-19 for each
+# other family, all five folds together; the training rows of any one fold hold fewer. So arm B cannot offer a BLOCK in
+# any family by construction, and the expected verdict is inconclusive with arm B flagging nothing. That is the finding
+# this run would confirm: on this data the bound, not the calibration gate, is what makes BLOCK unreachable. Arm C (point precision >= 0.95 over >= 10 training flags) is a secondary,
+# non-deciding arm that measures how far the held-out precision is from the bar when the bound is relaxed.
+id: exp-003-precision-bound-blocking
+hypothesis: >
+  The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4,
+  which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training
+  folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision
+  bar on held-out repositories where the calibrated rule offers no BLOCK.
+kind: rule
+programs:                   # the slice-2 incumbents (benchmarks/measurements/2026-10-02-decision-engine-slice-2, program.id)
+  access_control: 745cb1d76bbb09dc1574d333d2118489be62e39886517894d3699fa565b0e8c2
+  deserialization: 3c0c2831be7d78198a62351661c37867363c501cb40e3ed7b0119c155af09654
+  injection: b207a0710e0abf053b3b9397b25f9ee1cab4679144865e3a82c1b9be2c7b4982
+  output_encoding: 5d768f16724562c381c09949ff3df9cbb11d58cbb7a9c0515d0dc3f858d3818e
+  path: aea396680a4874dcb6de70e240ca0480c83424debdd3710bc3f1ea6a612661f3
+  untrusted_destination: 1242b3973fc89056aa08e00843043d68235df217d8e3ef8a0ea0e0b81c142753
+arms:
+  A:
+    label: today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p
+    rule: {name: calibrated_block}
+  B:
+    label: selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95
+    rule: {name: precision_bound_block, bound: wilson, precision: 0.95, select_on: paired}
+  C:
+    label: secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags
+    rule: {name: precision_bound_block, bound: point, precision: 0.95, min_flagged: 10, select_on: paired}
+unit: candidate
+pairing: paired             # every unit scored once; every arm decides every unit
+cluster: repository
+order: {randomise: none, seed: 3}
+# primary population: paired units (both sides of the candidate present). It excludes every positives-only group and any
+# unpaired unit, whose positives never had to be told apart from their own fix. Pooled (every unit) is reported beside it.
+populations: {primary: paired, secondary: [pooled]}
+metric:
+  primary:
+    - {name: flagged_precision_lb, target: 0.95}   # held-out (outer-fold) Wilson 95% LB of flagged precision, per family, arm B, paired
+  secondary: [flagged_precision, coverage, flagged]  # coverage = flagged true positives / positives
+# Decision rule (decide_rule): REJECT B if in any family B flags held-out paired units at a point precision < 0.95;
+# else ADOPT B if in some family B's held-out paired Wilson LB >= 0.95; else INCONCLUSIVE (the incumbent rule stays).
+# A family "reaches the M4 precision bar on held-out repositories" when its held-out paired Wilson LB >= 0.95 (the
+# point precision is reported beside it).
+decision: {arm: B, target_precision: 0.95}
+test: {resamples: 2000}     # repository-cluster bootstrap of the held-out flagged precision, beside the Wilson bound
+mde: 0.0                    # not a paired difference test; see the power note above
+budget_usd: {total: 0}      # zero cost is a hard rule
+families: [access_control, deserialization, injection, output_encoding, path, untrusted_destination]
+folds: {kind: outer, k: 5, seed: 0, compile_fraction: 0.25}   # the incumbent evaluation's folds (learn/compile.toml seed)
+units_file: plane/experiments/exp-003-precision-bound-blocking.units.jsonl   # frozen by `ousast learn experiment units`

src/openultrasast/learn/cli.py

@@ -122,11 +122,17 @@ def _experiment(args: argparse.Namespace) -> int:
             print(json.dumps({k: row[k] for k in shown}, indent=2))
             return 0
         if args.experiment_command == "analyse":
-            row = ex.check_registered(store, manifest, need_units=False)
+            row = ex.check_registered(store, manifest, need_units=manifest.kind == "rule")
             provenance = {"code_commit": ex.committed_at_head(manifest.path)}
-            report = ex.analyse(manifest, ex.load_outcomes(store, manifest.id), resamples=args.resamples, provenance=provenance)
+            outcomes = ex.load_outcomes(store, manifest.id)
+            if manifest.kind == "rule":
+                units = ex.read_units(manifest.units_file)
+                report = ex.analyse_rule(manifest, outcomes, units, resamples=args.resamples, provenance=provenance)
+            else:
+                report = ex.analyse(manifest, outcomes, resamples=args.resamples, provenance=provenance)
             if args.record:
-                store.put_row(ex.result_row(manifest, str(row["pin"]), report))
+                result = ex.rule_result_row if manifest.kind == "rule" else ex.result_row
+                store.put_row(result(manifest, str(row["pin"]), report))
             _write(report, args.out or (ex.RESULT_DIR / manifest.id / "result.json" if args.record else None))
             return 0
         from .embeddings import EmbeddingCache
@@ -143,6 +149,16 @@ def _experiment(args: argparse.Namespace) -> int:
         cache = EmbeddingCache(store)
         vectors = {sha: v for sha in cache.names() if (v := cache.get(sha)) is not None} or None
         parameters = _model_parameters(args.manifest_model)
+
+        def excerpt_text(sha: str) -> str | None:
+            data = store.get_blob("excerpts", sha)
+            return data.decode("utf-8") if data is not None else None
+
+        if manifest.kind == "rule":  # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved
+            caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store)
+            summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors)
+            _write(summary.as_dict(), args.out)
+            return 0
         callers: dict[str, Caller] = {}
         for name, arm in manifest.arms.items():
             if arm.replay_only:
@@ -157,10 +173,6 @@ def _experiment(args: argparse.Namespace) -> int:
             client = MeteredClient(resolved[0], prices=parameters, budget_usd=arm.budget_usd)
             callers[name] = Caller(client, _arm_model(manifest, arm, store), parameters, store=store)
 
-        def excerpt_text(sha: str) -> str | None:
-            data = store.get_blob("excerpts", sha)
-            return data.decode("utf-8") if data is not None else None
-
         early = not args.no_early_stop
         summary = ex.run(store, manifest, memory, callers, excerpt_text, vectors, stop_early=early, resamples=args.resamples)
     except ex.ExperimentError as exc:

src/openultrasast/learn/evaluate.py

@@ -54,6 +54,7 @@ class Prediction:
     parse_failed: int = 0
     p: float | None = None
     point: str | None = None  # BLOCK | ADVISORY | DROP, from points chosen on the other folds
+    pair: str = ""  # the two sides of one candidate share it; "" when only one side is present
 
     @property
     def blockable(self) -> bool:
@@ -128,6 +129,87 @@ def nested_points(preds: Sequence[Prediction]) -> tuple[list[Prediction], dict[s
     return out, points
 
 
+BOUNDS = ("wilson", "point")
+
+
+def precision_threshold(
+    rows: Sequence[Prediction], *, precision: float = TARGET_PRECISION, bound: str = "wilson", min_flagged: int = 1
+) -> float | None:
+    """The lowest raw-score threshold ``t`` at which the blockable rows with ``s >= t`` reach ``precision``: their
+    Wilson 95% lower bound (``bound="wilson"``) or their point precision over at least ``min_flagged`` rows
+    (``bound="point"``). Candidate thresholds are the rows' own scores; ``None`` when no threshold qualifies."""
+    if bound not in BOUNDS:
+        raise ValueError(f"bound {bound!r} (one of {', '.join(BOUNDS)})")
+    blockable = [r for r in rows if r.blockable]
+    for t in sorted({r.s for r in blockable}):
+        chosen = [r for r in blockable if r.s >= t]
+        hits = sum(r.label for r in chosen)
+        if len(chosen) < max(1, min_flagged):
+            continue
+        level = wilson_lower(hits, len(chosen)) if bound == "wilson" else hits / len(chosen)
+        if level >= precision:
+            return t
+    return None
+
+
+def nested_precision_block(
+    preds: Sequence[Prediction],
+    *,
+    eligible: Callable[[Prediction], bool] | None = None,
+    precision: float = TARGET_PRECISION,
+    bound: str = "wilson",
+    min_flagged: int = 1,
+) -> tuple[list[Prediction], dict[str, dict[str, float | None]]]:
+    """Selective blocking on the raw score, per family: the threshold for outer fold j is chosen on the *other* folds'
+    rows of the family only (and, with ``eligible``, only those rows); the held-out fold's labels never enter it.
+    Every row of fold j is then decided with that threshold: ``BLOCK`` when blockable and ``s >= t``, else ``DROP``.
+    A family whose training folds reach no threshold offers no BLOCK on that fold. Returns the decided rows and the
+    thresholds by family and fold."""
+    thresholds: dict[str, dict[str, float | None]] = {}
+    for family in sorted({r.family for r in preds}):
+        rows = [r for r in preds if r.family == family]
+        thresholds[family] = {
+            fold: precision_threshold(
+                [r for r in rows if r.fold != fold and (eligible is None or eligible(r))],
+                precision=precision, bound=bound, min_flagged=min_flagged,
+            )
+            for fold in sorted({r.fold for r in rows})
+        }  # fmt: skip
+    out = []
+    for r in preds:
+        t = thresholds[r.family][r.fold]
+        out.append(replace(r, point="BLOCK" if t is not None and r.blockable and r.s >= t else "DROP"))
+    return out, thresholds
+
+
+def calibrated_block(preds: Sequence[Prediction], *, seed: int = 0, **floors: int) -> tuple[list[Prediction], dict[str, Any]]:
+    """Today's BLOCK rule, per family as the slice-2 evaluation applied it: the Platt map cross-fitted on the other
+    folds, the calibration check on the family's held-out calibrated predictions, the BLOCK point chosen on the other
+    folds' calibrated predictions (nested); a row is ``BLOCK`` only when the family's calibration holds. Returns the
+    decided rows and, per family, the check and the calibrated thresholds by fold."""
+    out: list[Prediction] = []
+    info: dict[str, Any] = {}
+    for family in sorted({r.family for r in preds}):
+        rows = [r for r in preds if r.family == family]
+        ps, _ = cross_fit([Row(r.fold, r.family, r.group, r.label, r.s) for r in rows], **floors)
+        calibrated = [replace(r, p=p) for r, p in zip(rows, ps, strict=True)]
+        decided = [r for r in calibrated if r.p is not None]
+        checked = (
+            check([float(r.p) for r in decided if r.p is not None], [r.label for r in decided], [r.group for r in decided], seed=seed)
+            if len({r.label for r in decided}) == 2
+            else None
+        )
+        holds = bool(checked and checked.holds)
+        nested, points = nested_points(calibrated)
+        out += [replace(r, point="BLOCK" if holds and r.point == "BLOCK" else "DROP") for r in nested]
+        info[family] = {
+            "calibration_holds": holds, "ece": None if checked is None else round(checked.ece, 4),
+            "thresholds_by_fold": {f: p.block for f, p in points.items()},
+            "reason_by_fold": {f: ("calibration" if not holds else p.block_reason) for f, p in points.items()},
+        }  # fmt: skip
+    return out, info
+
+
 def _at(rows: Sequence[Prediction], level: str) -> dict[str, float | None]:
     flagged = [r for r in rows if r.point in (("BLOCK",) if level == "block" else ("BLOCK", "ADVISORY"))]
     positives = sum(r.label for r in rows)
@@ -326,8 +408,9 @@ def candidate_from_example(
 
 
 __all__ = [
-    "PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "candidate_from_example", "choose_points",
+    "BOUNDS", "PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "calibrated_block",
-    "cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible", "learning_curve", "nested_points", "predict", "strata",
+    "candidate_from_example", "choose_points", "cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible",
-    "summary", "targets", "unevaluable_families",
+    "learning_curve", "nested_points", "nested_precision_block", "precision_threshold", "predict", "strata", "summary", "targets",
+    "unevaluable_families",
     "wilson_lower",
 ]  # fmt: skip

src/openultrasast/learn/experiments.py

@@ -41,10 +41,10 @@ from typing import Any
 
 import yaml
 
-from ..plane.budget import BudgetExhausted
+from ..plane.budget import BudgetExhausted, MeteredClient
 from ..plane.memory import MemoryStore, row_id
 from .compile import load_program, spec_of
-from .evaluate import candidate_from_example
+from .evaluate import BOUNDS, TARGET_PRECISION, Prediction, calibrated_block, candidate_from_example, nested_precision_block, wilson_lower
 from .examples import EXAMPLE_KIND, Example
 from .folds import Fold, compile_split, folds_digest, outer_folds
 from .program import Caller, Decision, Program, ProgramSpec
@@ -57,7 +57,15 @@ RESULT_KIND = "experiment_result"
 REGISTER_TASK = "register"
 POPULATION = "experiments"  # the envelope of every experiment row; `split` is the experiment id
 RUN_TASK = "run"
-ARM_KINDS = ("program", "plane")
+ARM_KINDS = ("program", "plane", "rule")
+# decision-rule experiments (kind: rule): one replay-only scoring pass of the incumbent programs, arms differ only in
+# the BLOCK rule applied to the same scores
+RULES = ("calibrated_block", "precision_bound_block")
+POPULATIONS = ("paired", "pooled")  # paired: units whose candidate has both sides present; pooled: every unit
+SCORE_ARM = "score"  # the arm name of a rule experiment's outcome rows (the shared scores)
+MISS_VERDICT = "replay_miss"  # an outcome row of a unit excluded because a response was not cached
+# rule metrics, per family and population of held-out (outer-fold) units: direction only (no paired test)
+RULE_METRICS: Mapping[str, int] = {"flagged_precision_lb": 1, "flagged_precision": 1, "coverage": 1, "flagged": 1}
 UNITS = ("candidate", "file", "repository")
 PAIRINGS = ("paired", "assigned")
 # O'Brien-Fleming boundaries for two equally spaced looks at alpha 0.05, two-sided (design section 6).
@@ -96,6 +104,7 @@ class Arm:
     replay_only: bool = False
     repeats: int = 1
     budget_usd: float = 0.0
+    rule: Mapping[str, Any] = field(default_factory=dict)  # rule arms: {name: calibrated_block | precision_bound_block, ...}
 
 
 @dataclass(frozen=True)
@@ -119,11 +128,11 @@ class Metric:
 
     @property
     def direction(self) -> int:
-        return METRICS[self.name][0]
+        return RULE_METRICS[self.name] if self.name in RULE_METRICS else METRICS[self.name][0]
 
     @property
     def binary(self) -> bool:
-        return METRICS[self.name][1]
+        return self.name in METRICS and METRICS[self.name][1]
 
     def as_dict(self) -> dict[str, Any]:
         return {k: v for k, v in asdict(self).items() if v is not None}
@@ -198,6 +207,7 @@ def load_manifest(path: Path) -> Manifest:
                 bool(spec.get("replay_only", False)),
                 int(spec.get("repeats", 1)),
                 float(spec.get("budget_usd", 0.0)),
+                dict(spec.get("rule") or {}),
             )  # fmt: skip
             for name, spec in dict(raw["arms"]).items()
         }
@@ -210,9 +220,12 @@ def load_manifest(path: Path) -> Manifest:
         primary_raw = metric["primary"]
         primary = tuple(Metric.parse(m) for m in (primary_raw if isinstance(primary_raw, list) else [primary_raw]))
         secondary = tuple(Metric.parse(m) for m in metric.get("secondary", []))
+        known = RULE_METRICS if kind == "rule" else METRICS
         for m in (*primary, *secondary):
-            if m.name not in METRICS:
+            if m.name not in known:
-                raise ExperimentError(f"{path}: metric {m.name!r} (one of {', '.join(METRICS)})")
+                raise ExperimentError(f"{path}: metric {m.name!r} (one of {', '.join(known)})")
+        if kind == "rule":
+            _check_rule_manifest(path, raw, arms)
         test = dict(raw.get("test") or {})
         stopping = dict(raw.get("stopping") or {"looks": [1.0], "boundaries": [1.959964]})
         looks = tuple(float(x) for x in stopping.get("looks", [0.5, 1.0]))
@@ -244,11 +257,39 @@ def load_manifest(path: Path) -> Manifest:
         raise ExperimentError(f"{path}: missing {exc}") from exc
     if manifest.id != path.stem:
         raise ExperimentError(f"{path}: id {manifest.id!r} must equal the file stem")
-    if kind == "program" and not manifest.families:
+    if kind in ("program", "rule") and not manifest.families:
-        raise ExperimentError(f"{path}: a program experiment names its families")
+        raise ExperimentError(f"{path}: a {kind} experiment names its families")
+    if kind == "rule" and set(manifest.families) - set(manifest.programs):
+        raise ExperimentError(f"{path}: a rule experiment names a program for every family (programs:)")
     return manifest
 
 
+def _check_rule_manifest(path: Path, raw: Mapping[str, Any], arms: Mapping[str, Arm]) -> None:
+    """A decision-rule experiment costs nothing by construction: no arm overrides the program, no budget, and every
+    arm names a known rule with known parameters; the populations are declared before the run."""
+    budget = raw["budget_usd"]
+    total = float(budget["total"] if isinstance(budget, Mapping) else budget)
+    if total != 0 or any(a.budget_usd for a in arms.values()):
+        raise ExperimentError(f"{path}: a rule experiment replays cached responses only: budget_usd must be 0")
+    for arm in arms.values():
+        if arm.spec or arm.task_env or arm.programs:
+            raise ExperimentError(f"{path}: arm {arm.name} of a rule experiment may change the BLOCK rule only (rule:)")
+        name = arm.rule.get("name")
+        if name not in RULES:
+            raise ExperimentError(f"{path}: arm {arm.name} rule.name {name!r} (one of {', '.join(RULES)})")
+        allowed = {"name"} | ({"precision", "bound", "min_flagged", "select_on"} if name == "precision_bound_block" else set())
+        if set(arm.rule) - allowed:
+            raise ExperimentError(f"{path}: arm {arm.name} rule parameters {sorted(set(arm.rule) - allowed)} are not understood")
+        if arm.rule.get("bound", "wilson") not in BOUNDS or arm.rule.get("select_on", "paired") not in POPULATIONS:
+            raise ExperimentError(f"{path}: arm {arm.name} bound in {BOUNDS} and select_on in {POPULATIONS}")
+    populations = dict(raw.get("populations") or {})
+    if populations.get("primary") not in POPULATIONS or any(p not in POPULATIONS for p in populations.get("secondary", [])):
+        raise ExperimentError(f"{path}: populations.primary (and secondary) in {POPULATIONS}")
+    decision = dict(raw.get("decision") or {})
+    if not 0 < float(decision.get("target_precision", 0)) <= 1 or decision.get("arm") not in arms:
+        raise ExperimentError(f"{path}: decision.target_precision in (0, 1] and decision.arm naming an arm")
+
+
 # --- register ------------------------------------------------------------------------------------------------------------
 
 
@@ -383,7 +424,7 @@ def program_folds(memory: Sequence[Example], manifest: Manifest) -> tuple[list[F
 def freeze_units(store: MemoryStore, manifest: Manifest, memory: Sequence[Example]) -> list[Unit]:
     """The evaluation candidates of the manifest's families under the incumbent's outer folds, paired by candidate
     across the two pins where both sides are in memory (no model; the list is then committed and registered)."""
-    if manifest.kind != "program":
+    if manifest.kind not in ("program", "rule"):
         raise ExperimentError(f"{manifest.id}: units of a plane experiment come from candidate generation, not the memory")
     folds, _ = program_folds(memory, manifest)
     sides: dict[str, tuple[str, str, str]] = {}
@@ -812,6 +853,8 @@ def run(
     """Every unit in every arm, paired, repositories in the pre-shuffled look order, a seeded coin per unit for which
     arm goes first; outcomes already in the store are not decided again. After the first look's repositories the
     primary is tested against its boundary and, when crossed, the run stops. A ceiling ends the run ``unfinished``."""
+    if manifest.kind == "rule":
+        raise ExperimentError(f"{manifest.id}: a rule experiment scores once at zero cost (run_rule), its arms differ only in the rule")
     if manifest.kind != "program":
         raise ExperimentError(f"{manifest.id}: plane arms run as plane Runs; their outcomes arrive as arm_outcome rows")
     row = check_registered(store, manifest)
@@ -892,7 +935,267 @@ def run(
     return summary
 
 
+# --- decision-rule experiments (kind: rule) -----------------------------------------------------------------------------
+
+
+class ZeroCostViolation(ExperimentError):
+    """A model call reached the client of a zero-cost run, or the meter recorded spend: the run is void."""
+
+
+class _RefusingChat:
+    """The inner client behind a zero-cost run's meter. The meter's zero ceilings stop every call before it gets here;
+    reaching it means a guard was bypassed."""
+
+    usage: list[Any] = []
+
+    def complete(self, **_: Any) -> Any:
+        raise ZeroCostViolation("a model call reached the client of a zero-cost run")
+
+
+def zero_cost_caller(model: str, prices: Mapping[str, Any], store: MemoryStore | None) -> tuple[Caller, MeteredClient]:
+    """A caller that answers only from the response cache: its client is a meter with zero ceilings (USD and calls)
+    around a client that refuses. A response that is not cached raises ``BudgetExhausted`` before any call; the run
+    excludes that unit and counts it. The meter is the proof: its ``calls`` and ``usd`` must stay 0."""
+    meter = MeteredClient(_RefusingChat(), prices=prices, budget_usd=0.0, budget_calls=0)  # type: ignore[arg-type]
+    return Caller(meter, model, prices, store=store), meter
+
+
+def _checked_units(manifest: Manifest, memory: Sequence[Example]) -> tuple[list[Unit], dict[str, Fold], str]:
+    units = read_units(manifest.units_file)
+    folds, fdigest = program_folds(memory, manifest)
+    by_name = {f.name: f for f in folds}
+    index = {e.id for e in memory}
+    for unit in units:
+        fold = by_name.get(unit.fold)
+        if fold is None or unit.group not in fold.eval_groups or unit.unit not in index:
+            raise ExperimentError(
+                f"{manifest.id}: unit {unit.unit[:12]} is not under fold {unit.fold!r} of this memory; the memory changed"
+            )
+    return units, by_name, fdigest
+
+
+def run_rule(
+    store: MemoryStore,
+    manifest: Manifest,
+    memory: Sequence[Example],
+    caller: Caller,
+    meter: MeteredClient,
+    excerpt_text: Callable[[str], str | None],
+    vectors: Mapping[str, Sequence[float]] | None = None,
+    *,
+    on_decision: Callable[[Outcome, Decision], None] | None = None,
+) -> RunSummary:
+    """The one scoring pass of a rule experiment: every unit decided once by its family's incumbent program, answered
+    only from the response cache. A unit whose responses are not all cached is excluded and counted (an outcome row
+    with ``verdict = replay_miss`` and no score), never asked. Raises :class:`ZeroCostViolation` when the meter shows
+    a call or any spend afterwards."""
+    if manifest.kind != "rule":
+        raise ExperimentError(f"{manifest.id}: run_rule runs decision-rule experiments only")
+    if caller.client is not meter or meter.budget_usd != 0.0 or meter.budget_calls != 0:
+        raise ZeroCostViolation(f"{manifest.id}: a rule experiment runs only behind a zero-ceiling meter (zero_cost_caller)")
+    row = check_registered(store, manifest)
+    units, by_name, fdigest = _checked_units(manifest, memory)
+    index = {e.id: e for e in memory}
+    programs = {f: Program(arm_spec(manifest, manifest.arms["A"], f, store), memory, excerpt_text, vectors) for f in manifest.families}
+    models = {spec_of(load_program(store, manifest.programs[f])).model for f in manifest.families}
+    if models != {caller.model}:
+        raise ExperimentError(f"{manifest.id}: the programs' models {sorted(models)} differ from the caller's {caller.model!r}")
+    candidate_of = candidate_from_example(excerpt_text, vectors)
+    have = {outcome_id(manifest.id, o.arm, o.unit, o.replicate) for o in load_outcomes(store, manifest.id)}
+    decided, skipped = {SCORE_ARM: 0}, {"already": 0, MISS_VERDICT: 0}
+    fold_seed = int(manifest.folds.get("seed", 0))
+    for unit in sorted(units, key=lambda u: (u.group, u.family, u.unit)):
+        if outcome_id(manifest.id, SCORE_ARM, unit.unit, 0) in have:
+            skipped["already"] += 1
+            continue
+        candidate = candidate_of(index[unit.unit])
+        if candidate is None:
+            raise ExperimentError(f"{manifest.id}: unit {unit.unit[:12]} has no excerpt in the store")
+        try:
+            decision = programs[unit.family].decide(candidate, by_name[unit.fold], caller, seed=fold_seed)
+        except BudgetExhausted:  # a response is not cached: the zero ceiling stopped the call before it started
+            outcome = Outcome(unit.unit, unit.group, unit.family, unit.label, unit.pair, SCORE_ARM, 0, None, MISS_VERDICT, 0.0)
+            skipped[MISS_VERDICT] += 1
+            store.put_row(outcome_row(manifest.id, str(row["pin"]), outcome))
+            continue
+        outcome = Outcome(unit.unit, unit.group, unit.family, unit.label, unit.pair, SCORE_ARM, 0, decision.s, decision.verdict, 0.0,
+                          decision.parse_failed)  # fmt: skip
+        store.put_row(outcome_row(manifest.id, str(row["pin"]), outcome))
+        decided[SCORE_ARM] += 1
+        if on_decision is not None:
+            on_decision(outcome, decision)
+    if meter.calls or (meter.usd or 0.0) > 0 or caller.calls:
+        raise ZeroCostViolation(f"{manifest.id}: the meter recorded {meter.calls} calls and ${meter.usd}; the run is void")
+    spend = {SCORE_ARM: {"usd": meter.usd, "client_calls": meter.calls, "replayed": caller.replayed, "cache_digest": caller.cache_digest(),
+                         "replay_priced_usd": caller.usd()}}  # fmt: skip
+    summary = RunSummary(manifest.id, "done", len(units), decided, skipped, spend, fdigest)
+    store.put_row({
+        "id": row_id(EXPERIMENT_KIND, f"experiment-{manifest.id}", RUN_TASK, manifest.id), "kind": EXPERIMENT_KIND,
+        "repo": experiment_repo(manifest.id), "pin": str(row["pin"]), "run": f"experiment-{manifest.id}", "task": RUN_TASK,
+        "population": POPULATION, "split": manifest.id, "image": "host", "experiment": manifest.id,
+        "manifest_digest": manifest.digest, **summary.as_dict(),
+    })  # fmt: skip
+    return summary
+
+
+def min_flagged_for_bound(target: float, z: float = 1.959964) -> int:
+    """The fewest flagged units, all correct, whose Wilson lower bound reaches ``target`` (73 at 0.95)."""
+    n = 1
+    while wilson_lower(n, n, z) < target:
+        n += 1
+    return n
+
+
+def _group_bootstrap_precision(rows: Sequence[Prediction], *, resamples: int, seed: int) -> list[float] | None:
+    by_group: dict[str, list[Prediction]] = {}
+    for r in rows:
+        by_group.setdefault(r.group, []).append(r)
+    names = sorted(by_group)
+    rng = random.Random(seed)
+    values = []
+    for _ in range(resamples if names else 0):
+        sample = [r for g in (rng.choice(names) for _ in names) for r in by_group[g]]
+        flagged = [r for r in sample if r.point == "BLOCK"]
+        if flagged:
+            values.append(sum(r.label for r in flagged) / len(flagged))
+    if len(values) < 2:
+        return None
+    values.sort()
+    return [values[int(0.025 * (len(values) - 1))], values[int(0.975 * (len(values) - 1))]]
+
+
+def flag_summary(rows: Sequence[Prediction], target: float, *, resamples: int = 2000, seed: int = 0) -> dict[str, Any]:
+    """Held-out BLOCK decisions of one population: flagged count, flagged precision with its Wilson 95% lower bound
+    and a repository-cluster bootstrap interval, coverage (the share of positives flagged), and the M4 reading."""
+    flagged = [r for r in rows if r.point == "BLOCK"]
+    hits = sum(r.label for r in flagged)
+    positives = sum(r.label for r in rows)
+    precision = hits / len(flagged) if flagged else None
+    lower = wilson_lower(hits, len(flagged)) if flagged else None
+    return {
+        "units": len(rows), "groups": len({r.group for r in rows}), "positives": positives, "flagged": len(flagged),
+        "true_flagged": hits, "flagged_precision": None if precision is None else round(precision, 4),
+        "flagged_precision_lb": None if lower is None else round(lower, 4),
+        "flagged_precision_ci": _group_bootstrap_precision(rows, resamples=resamples, seed=seed) if flagged else None,
+        "coverage": round(hits / positives, 4) if positives else None,
+        "meets_target_point": precision is not None and precision >= target,
+        "meets_target_lb": lower is not None and lower >= target,
+    }  # fmt: skip
+
+
+def _apply_rule(arm: Arm, preds: Sequence[Prediction], seed: int) -> tuple[list[Prediction], dict[str, Any]]:
+    if arm.rule["name"] == "calibrated_block":
+        return calibrated_block(preds, seed=seed)
+    select_on = str(arm.rule.get("select_on", "paired"))
+    decided, thresholds = nested_precision_block(
+        preds, eligible=(lambda r: bool(r.pair)) if select_on == "paired" else None,
+        precision=float(arm.rule.get("precision", TARGET_PRECISION)), bound=str(arm.rule.get("bound", "wilson")),
+        min_flagged=int(arm.rule.get("min_flagged", 1)),
+    )  # fmt: skip
+    return decided, {f: {"thresholds_by_fold": t} for f, t in thresholds.items()}
+
+
+def decide_rule(manifest: Manifest, families: Mapping[str, Mapping[str, Any]]) -> tuple[str, str]:
+    """The pre-registered rule: **reject** the decision arm when it flags held-out units of the primary population in
+    some family at a point precision below the target; **adopt** it when it does not and some family's held-out
+    Wilson lower bound reaches the target; otherwise **inconclusive** (the incumbent rule stays)."""
+    decision = dict(manifest.raw["decision"])
+    arm, target = str(decision["arm"]), float(decision["target_precision"])
+    primary = str(manifest.raw["populations"]["primary"])
+    cells = {f: v[arm][primary] for f, v in families.items()}
+    low = sorted(f for f, c in cells.items() if c["flagged"] and not c["meets_target_point"])
+    if low:
+        return "reject", f"{arm} flags held-out {primary} units below precision {target} in {', '.join(low)}"
+    reached = sorted(f for f, c in cells.items() if c["meets_target_lb"])
+    if reached:
+        return "adopt", f"{arm}'s held-out Wilson lower bound reaches {target} in {', '.join(reached)} and nowhere flags below it"
+    if not any(c["flagged"] for c in cells.values()):
+        return "inconclusive", f"{arm} offers no BLOCK on held-out {primary} units in any family; the incumbent stays"
+    return "inconclusive", f"{arm} flags only at a Wilson lower bound below {target}; the incumbent stays"
+
+
+def analyse_rule(
+    manifest: Manifest,
+    outcomes: Sequence[Outcome],
+    units: Sequence[Unit],
+    *,
+    resamples: int | None = None,
+    provenance: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Counts and intervals only: every arm's rule applied to the same held-out scores, per family and population,
+    the pooled-over-families totals, the exclusions, the training capacity of each family against the Wilson bound,
+    and the pre-registered verdict."""
+    n = resamples if resamples is not None else min(manifest.resamples, 2000)
+    target = float(manifest.raw["decision"]["target_precision"])
+    populations = [str(manifest.raw["populations"]["primary"]), *[str(p) for p in manifest.raw["populations"].get("secondary", [])]]
+    by_unit = {u.unit: u for u in units}
+    scores = {o.unit: o for o in outcomes if o.arm == SCORE_ARM and o.replicate == 0 and o.unit in by_unit}
+    preds = [
+        Prediction(by_unit[u].fold, o.family, o.group, o.label, float(o.s), o.verdict, parse_failed=o.parse_failed, pair=o.pair)
+        for u, o in sorted(scores.items())
+        if o.s is not None
+    ]  # fmt: skip
+    excluded = {f: sum(1 for o in scores.values() if o.s is None and o.family == f) for f in manifest.families}
+    not_run = {f: sum(1 for u in units if u.family == f and u.unit not in scores) for f in manifest.families}
+
+    def select(rows: Sequence[Prediction], population: str) -> list[Prediction]:
+        return [r for r in rows if r.pair] if population == "paired" else list(rows)
+
+    decided = {name: _apply_rule(arm, preds, manifest.seed) for name, arm in manifest.arms.items()}
+    families: dict[str, dict[str, Any]] = {}
+    for family in manifest.families:
+        cell: dict[str, Any] = {}
+        for name, (rows, info) in decided.items():
+            mine = [r for r in rows if r.family == family]
+            cell[name] = {**info.get(family, {}), **{p: flag_summary(select(mine, p), target, resamples=n, seed=manifest.seed)
+                                                      for p in populations}}  # fmt: skip
+        mine = [r for r in preds if r.family == family]
+        folds = sorted({r.fold for r in mine})
+        cell["capacity"] = {
+            "evaluated": len(mine), "excluded_replay_miss": excluded[family], "not_run": not_run[family],
+            "paired_positives_by_training_folds": {f: sum(r.label for r in mine if r.pair and r.fold != f) for f in folds},
+        }  # fmt: skip
+        families[family] = cell
+    overall = {
+        name: {p: flag_summary(select(rows, p), target, resamples=n, seed=manifest.seed) for p in populations}
+        for name, (rows, _) in decided.items()
+    }
+    verdict, reason = decide_rule(manifest, families)
+    spend = {SCORE_ARM: round(sum(o.usd or 0.0 for o in outcomes if o.arm == SCORE_ARM), 6)}
+    return {
+        "experiment": manifest.id, "manifest_digest": manifest.digest, "hypothesis": manifest.hypothesis,
+        "arms": {a: {"label": arm.label, "rule": dict(arm.rule)} for a, arm in manifest.arms.items()},
+        "units": {"frozen": len(units), "evaluated": len(preds), "excluded_replay_miss": sum(excluded.values()),
+                  "not_run": sum(not_run.values()), "paired_evaluated": sum(1 for r in preds if r.pair),
+                  "groups": len({r.group for r in preds}), "by_label": {str(lab): sum(r.label == lab for r in preds) for lab in (0, 1)},
+                  "unsure": sum(1 for r in preds if not r.blockable)},
+        "populations": {"primary": populations[0], "secondary": populations[1:]}, "target_precision": target,
+        "min_flagged_for_bound": min_flagged_for_bound(target), "primary": [m.as_dict() for m in manifest.primary],
+        "secondary": [m.as_dict() for m in manifest.secondary], "families": families, "overall": overall, "resamples": n,
+        "decision": verdict, "reason": reason, "spend_usd": spend, **dict(provenance or {}),
+    }  # fmt: skip
+
+
+def rule_result_row(manifest: Manifest, pin: str, result: Mapping[str, Any]) -> dict[str, Any]:
+    """The ``experiment_result`` row of a rule experiment, with the column types of :func:`result_row`."""
+    primary = result["populations"]["primary"]
+    tests = {
+        f: {a: {k: cell[a][primary][k] for k in ("flagged", "flagged_precision", "flagged_precision_lb", "coverage")}
+            for a in manifest.arms}
+        for f, cell in result["families"].items()
+    }  # fmt: skip
+    return {
+        "id": row_id(RESULT_KIND, f"experiment-{manifest.id}", "analyse", manifest.id), "kind": RESULT_KIND,
+        "repo": experiment_repo(manifest.id), "pin": pin, "run": f"experiment-{manifest.id}", "task": "analyse", "population": POPULATION,
+        "split": manifest.id, "image": "host", "experiment": manifest.id, "manifest_digest": manifest.digest,
+        "decision": result["decision"], "reason": result["reason"], "unit_counts": result["units"], "spend_usd": result["spend_usd"],
+        "tests": tests, "looks_taken": [],
+    }  # fmt: skip
+
+
 __all__ = [
+    "MISS_VERDICT", "POPULATIONS", "RULES", "RULE_METRICS", "SCORE_ARM", "ZeroCostViolation", "analyse_rule", "decide_rule",
+    "flag_summary", "min_flagged_for_bound", "rule_result_row", "run_rule", "zero_cost_caller",
     "ALPHA", "ARM_KINDS", "EXPERIMENT_KIND", "MANIFEST_DIR", "METRICS", "OBF_TWO_LOOKS", "OUTCOME_KIND", "RESAMPLES", "RESULT_DIR",
     "RESULT_KIND", "TOST_MARGIN", "VERDICTS", "Arm", "ArmUnit", "Estimate", "ExperimentError", "Look", "Manifest", "McNemar", "Metric",
     "Outcome", "RunSummary", "Unit", "analyse", "arm_spec", "arm_units", "auc", "check_registered", "committed_at_head", "decide",

tests/test_learn_rules.py

@@ -0,0 +1,247 @@
+"""Decision-rule experiments (kind: rule): selective blocking on the raw score, chosen on training folds only, against
+today's calibrated BLOCK rule, scored once from the response cache at zero cost (exp-003)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from test_learn_experiments import CODE, EXAMPLES, PRICES, _json_shape, artifact, committed, example_rows, excerpt_text, paid, put_program
+
+from openultrasast.learn import experiments as ex
+from openultrasast.learn.evaluate import Prediction, calibrated_block, nested_precision_block, precision_threshold, wilson_lower
+from openultrasast.learn.program import Candidate, Program
+from openultrasast.learn.retrieve import Target
+from openultrasast.plane.memory import FileStore
+
+ROOT = Path(__file__).resolve().parents[1]
+
+RULE_MANIFEST = """
+id: {id}
+hypothesis: selective blocking beats the calibrated rule
+kind: rule
+programs: {{injection: {pid}, path: {pid}}}
+arms:
+  A: {{label: today, rule: {{name: calibrated_block}}}}
+  B: {{label: bound, rule: {{name: precision_bound_block, bound: wilson, precision: 0.95, select_on: paired}}}}
+  C: {{label: point, rule: {{name: precision_bound_block, bound: point, precision: 0.95, min_flagged: 2, select_on: paired}}}}
+populations: {{primary: paired, secondary: [pooled]}}
+decision: {{arm: B, target_precision: 0.95}}
+metric:
+  primary: [{{name: flagged_precision_lb, target: 0.95}}]
+  secondary: [flagged_precision, coverage, flagged]
+test: {{resamples: 50}}
+budget_usd: {{total: 0}}
+families: [injection, path]
+folds: {{k: 5, seed: 0, compile_fraction: 0.25}}
+units_file: {units}
+"""
+
+
+def write_rule_manifest(tmp_path: Path, pid: str, name: str = "exp-901-rule") -> Path:
+    path = tmp_path / f"{name}.yaml"
+    path.write_text(RULE_MANIFEST.format(id=name, pid=pid, units=tmp_path / f"{name}.units.jsonl"))
+    return path
+
+
+def pred(fold: int, label: int, s: float, *, family: str = "injection", group: str = "", verdict: str = "", pair: str = "p") -> Prediction:
+    verdict = verdict or ("vulnerable" if s >= 0.5 else "not_vulnerable")
+    return Prediction(f"outer-{fold}", family, group or f"org/r{fold}", label, s, verdict, pair=pair)
+
+
+# --- the threshold ---------------------------------------------------------------------------------------------------------
+
+
+def test_the_wilson_bound_needs_73_clean_flags_at_095() -> None:
+    assert ex.min_flagged_for_bound(0.95) == 73
+    assert wilson_lower(72, 72) < 0.95 <= wilson_lower(73, 73)
+    clean = [pred(0, 1, 0.9) for _ in range(73)] + [pred(0, 0, 0.2) for _ in range(10)]
+    assert precision_threshold(clean) == 0.9
+    assert precision_threshold(clean[1:]) is None  # 72 clean flags never reach the bound
+    assert precision_threshold(clean[1:], bound="point", min_flagged=10) == 0.9
+    assert precision_threshold(clean[:5], bound="point", min_flagged=10) is None
+
+
+def test_the_lowest_qualifying_threshold_and_unsure_never_blocks() -> None:
+    rows = [pred(0, 1, 0.9) for _ in range(80)] + [pred(0, 1, 0.4) for _ in range(5)]
+    assert precision_threshold(rows) == 0.4  # 85 clean flags at 0.4: the lowest threshold that qualifies
+    assert precision_threshold([*rows, pred(0, 0, 0.6)]) == 0.9  # one false flag at 0.6: 85 of 86 and 80 of 81 do not
+    unsure = [pred(0, 1, 0.95, verdict="unsure") for _ in range(100)]
+    assert precision_threshold(unsure) is None
+    decided, _ = nested_precision_block([*unsure[:3], *(pred(1, 1, 0.95) for _ in range(80))])
+    assert all(r.point == "DROP" for r in decided if r.verdict == "unsure")
+
+
+def leaky_threshold(preds: list[Prediction], fold: str) -> float | None:
+    """What a rule that peeked at the held-out fold would choose: every fold's rows, the reported fold included."""
+    return precision_threshold([r for r in preds if r.family == "injection"])
+
+
+def test_no_threshold_is_chosen_with_outer_fold_labels() -> None:
+    """Fold outer-0 holds 80 high-scoring positives; the other folds alone never reach the bound. A rule that used
+    outer-0's labels would block on outer-0; the nested rule must not, and flipping outer-0's labels must not move
+    outer-0's threshold."""
+    held = [pred(0, 1, 0.9, group=f"org/h{i}") for i in range(80)]
+    train = [pred(f, lab, 0.9 if lab else 0.3, group=f"org/t{f}-{i}") for f in range(1, 5) for i, lab in enumerate((1, 1, 1, 0, 0))]
+    preds = held + train
+    assert leaky_threshold(preds, "outer-0") is not None  # the test has teeth: the outer labels would qualify
+    decided, thresholds = nested_precision_block(preds)
+    assert thresholds["injection"]["outer-0"] is None
+    assert all(r.point == "DROP" for r in decided if r.fold == "outer-0")
+    flipped = [Prediction(r.fold, r.family, r.group, 1 - r.label, r.s, r.verdict, pair=r.pair) if r.fold == "outer-0" else r for r in preds]
+    _, again = nested_precision_block(flipped)
+    assert again["injection"]["outer-0"] == thresholds["injection"]["outer-0"]
+    # and the other folds' thresholds do see outer-0 (it is their training data)
+    assert all(thresholds["injection"][f"outer-{f}"] == 0.9 for f in range(1, 5))
+    assert all(again["injection"][f"outer-{f}"] is None for f in range(1, 5))
+
+
+def test_select_on_paired_ignores_unpaired_training_rows() -> None:
+    unpaired = [pred(1, 1, 0.9, pair="", group=f"org/u{i}") for i in range(80)]
+    paired = [pred(1, lab, 0.9 if lab else 0.1, group=f"org/p{i}") for i, lab in enumerate((1, 0) * 3)]
+    held = [pred(0, 1, 0.9), pred(0, 0, 0.1)]
+    _, everyone = nested_precision_block([*unpaired, *paired, *held])
+    _, only_paired = nested_precision_block([*unpaired, *paired, *held], eligible=lambda r: bool(r.pair))
+    assert everyone["injection"]["outer-0"] == 0.9 and only_paired["injection"]["outer-0"] is None
+
+
+def test_the_calibrated_rule_blocks_only_where_calibration_holds(monkeypatch: pytest.MonkeyPatch) -> None:
+    rows = [pred(f, 1 if i < 30 else 0, 0.9 if i < 30 else 0.1, group=f"org/g{f}-{i}") for f in range(5) for i in range(40)]
+    import openultrasast.learn.evaluate as ev
+
+    real = ev.check
+    for holds in (False, True):
+        monkeypatch.setattr(ev, "check", lambda *a, _h=holds, **k: ev.replace(real(*a, **k), holds=_h))
+        decided, info = calibrated_block(rows)
+        assert info["injection"]["calibration_holds"] is holds
+        blocked = [r for r in decided if r.point == "BLOCK"]
+        assert bool(blocked) is holds and all(r.label == 1 for r in blocked)
+
+
+# --- the manifest ------------------------------------------------------------------------------------------------------------
+
+
+def test_rule_manifests_cost_nothing_and_change_only_the_rule(tmp_path: Path) -> None:
+    path = write_rule_manifest(tmp_path, "0" * 64)
+    manifest = ex.load_manifest(path)
+    assert manifest.kind == "rule" and manifest.arms["B"].rule["bound"] == "wilson" and manifest.budget_total_usd == 0
+    text = path.read_text()
+    for bad, why in (
+        (text.replace("budget_usd: {total: 0}", "budget_usd: {total: 1}"), "budget_usd must be 0"),
+        (text.replace("A: {label: today, ", "A: {label: today, spec: {sampling: retrieval_ensemble}, "), "BLOCK rule only"),
+        (text.replace("name: calibrated_block", "name: magic"), "rule.name"),
+        (text.replace("bound: wilson", "bound: hope"), "bound in"),
+        (text.replace("primary: paired", "primary: all"), "populations"),
+        (text.replace("arm: B,", "arm: Z,"), "decision"),
+        (text.replace("flagged_precision_lb", "auc"), "metric"),
+        (text.replace("programs: {injection: " + "0" * 64 + ", path: ", "programs: {path: "), "program for every family"),
+    ):
+        path.write_text(bad)
+        with pytest.raises(ex.ExperimentError, match=why):
+            ex.load_manifest(path)
+
+
+# --- the zero-cost run ------------------------------------------------------------------------------------------------------
+
+
+def prepared(tmp_path: Path) -> tuple[FileStore, ex.Manifest, list[ex.Unit]]:
+    store = FileStore(tmp_path / "memory")
+    store.put_rows(example_rows())
+    pid = put_program(store, artifact())
+    manifest = ex.load_manifest(write_rule_manifest(tmp_path, pid))
+    units = ex.freeze_units(store, manifest, EXAMPLES)
+    ex.write_units(manifest.units_file, units)
+    ex.register(store, manifest, commit_of=committed)
+    return store, manifest, units
+
+
+def test_the_run_replays_the_cache_excludes_misses_and_proves_zero_spend(tmp_path: Path) -> None:
+    store, manifest, units = prepared(tmp_path)
+    chat, caller = paid()
+    caller.store = store
+    folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]}
+    index = {e.id: e for e in EXAMPLES}
+    cached = units[: len(units) - 3]  # the incumbent's evaluation answered all but three units
+    for u in cached:
+        e = index[u.unit]
+        program = Program(ex.arm_spec(manifest, manifest.arms["A"], u.family, store), EXAMPLES, excerpt_text)
+        program.decide(Candidate(Target.of(e), CODE[e.excerpt_sha], "python"), folds[u.fold], caller, seed=0)
+    paid_calls = len(chat.calls)
+    with pytest.raises(ex.ZeroCostViolation):  # a paid caller is refused before anything is decided
+        ex.run_rule(store, manifest, EXAMPLES, caller, caller.client, excerpt_text)
+    zero, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store)
+    summary = ex.run_rule(store, manifest, EXAMPLES, zero, meter, excerpt_text)
+    assert len(chat.calls) == paid_calls and meter.calls == 0 and meter.usd == 0.0 and zero.calls == 0
+    assert summary.decided == {"score": len(cached)} and summary.skipped == {"already": 0, "replay_miss": 3}
+    assert summary.spend["score"]["client_calls"] == 0 and summary.spend["score"]["usd"] == 0.0
+    outcomes = ex.load_outcomes(store, manifest.id)
+    assert sum(o.verdict == ex.MISS_VERDICT and o.s is None for o in outcomes) == 3
+    report = ex.analyse_rule(manifest, outcomes, units, resamples=20, provenance={"code_commit": "c" * 40})
+    assert report["units"]["excluded_replay_miss"] == 3 and report["units"]["evaluated"] == len(cached)
+    assert report["spend_usd"] == {"score": 0.0} and report["decision"] in ex.VERDICTS
+    assert set(report["families"]["injection"]) == {"A", "B", "C", "capacity"} and report["min_flagged_for_bound"] == 73
+    text = json.dumps(report)
+    assert "org0/repo0" not in text and units[0].unit not in text  # counts only
+    store.put_row(ex.rule_result_row(manifest, "c" * 40, report))
+    shapes: dict[str, set[str]] = {}  # one JSON type per column across the experiment's rows (S3 Select, see test_learn_experiments)
+    for record in store.rows(repo=ex.experiment_repo(manifest.id)):
+        for name, value in record.row.items():
+            if value is not None:
+                shapes.setdefault(name, set()).add(_json_shape(value))
+    assert {name: sorted(s) for name, s in shapes.items() if len(s) > 1} == {}
+    again = ex.run_rule(store, manifest, EXAMPLES, *ex.zero_cost_caller("deepseek-flash", PRICES, store), excerpt_text)
+    assert again.decided == {"score": 0} and again.skipped["already"] == len(units)
+    with pytest.raises(ex.ExperimentError, match="zero cost"):
+        ex.run(store, manifest, EXAMPLES, {}, excerpt_text)
+
+
+def test_a_call_past_the_meter_voids_the_run() -> None:
+    caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, None)
+    with pytest.raises(ex.ZeroCostViolation):
+        meter.inner.complete(model="deepseek-flash", messages=[], tools=[])
+
+
+# --- the analysis --------------------------------------------------------------------------------------------------------------
+
+
+def crafted(tmp_path: Path, *, b_clean: bool) -> tuple[ex.Manifest, list[ex.Outcome], list[ex.Unit]]:
+    """Five folds of 20 candidate pairs each: positives at s 0.9; negatives at 0.1, except four per fold at 0.9 unless
+    ``b_clean``."""
+    manifest = ex.load_manifest(write_rule_manifest(tmp_path, "0" * 64))
+    units, outcomes = [], []
+    for f in range(5):
+        for i in range(40):
+            label = 1 if i < 20 else 0
+            high = label == 1 or (not b_clean and i >= 36)
+            u = ex.Unit(f"u{f}-{i}", f"org/r{f}-{i % 20}", "injection", label, f"outer-{f}", f"p{f}-{i % 20}")
+            units.append(u)
+            s = 0.9 if high else 0.1
+            outcomes.append(ex.Outcome(u.unit, u.group, u.family, label, u.pair, ex.SCORE_ARM, 0, s,
+                                       "vulnerable" if high else "not_vulnerable", 0.0))  # fmt: skip
+    return manifest, outcomes, units
+
+
+def test_the_verdict_follows_the_registered_rule(tmp_path: Path) -> None:
+    manifest, outcomes, units = crafted(tmp_path, b_clean=True)
+    report = ex.analyse_rule(manifest, outcomes, units, resamples=20)
+    cell = report["families"]["injection"]["B"]["paired"]
+    assert cell["flagged"] == 100 and cell["flagged_precision"] == 1.0 and cell["coverage"] == 1.0 and cell["meets_target_lb"]
+    assert report["decision"] == "adopt"
+    assert report["families"]["path"]["B"]["paired"]["flagged"] == 0  # a family without units flags nothing
+    manifest, outcomes, units = crafted(tmp_path, b_clean=False)
+    report = ex.analyse_rule(manifest, outcomes, units, resamples=20)
+    assert report["families"]["injection"]["B"]["paired"]["flagged"] == 0 and report["decision"] == "inconclusive"
+    assert report["families"]["injection"]["C"]["paired"]["flagged"] == 0  # 80/96 point precision never reaches 0.95 either
+    row = ex.rule_result_row(manifest, "c" * 40, report)
+    assert row["kind"] == "experiment_result" and row["tests"]["injection"]["B"]["flagged"] == 0 and row["looks_taken"] == []
+
+
+def test_the_registered_exp003_manifest_loads() -> None:
+    path = ROOT / "plane/experiments/exp-003-precision-bound-blocking.yaml"
+    if not path.exists():
+        pytest.skip("exp-003 is not written yet")
+    manifest = ex.load_manifest(path)
+    assert manifest.kind == "rule" and manifest.budget_total_usd == 0 and manifest.raw["populations"]["primary"] == "paired"
+    assert manifest.arms["A"].rule == {"name": "calibrated_block"} and manifest.arms["B"].rule["bound"] == "wilson"
+    assert len(manifest.families) == 6 and set(manifest.programs) == set(manifest.families)