Open_Ultra_SAST/benchmarks/measurements/2026-09-06-candidate-ceiling-all-slices.json

{
  "command": "uv run ousast learning candidates --slice all",
  "deciding_slices": {
    "agent-vfc": {
      "ceiling": 0.897,
      "headroom": 0.453,
      "round_zero_detection": 0.444,
      "verdict": "go"
    },
    "github": {
      "ceiling": 1.0,
      "round_zero_detection": null,
      "verdict": "go"
    },
    "vfc": {
      "ceiling": 0.847,
      "note": "baseline still running; non-gating",
      "round_zero_detection": null,
      "verdict": "go"
    },
    "vfc-js": {
      "ceiling": 0.529,
      "note": "eight of seventeen rows label a function the JavaScript CST names `<anon>`: `module.exports.x = function () {}` and friends. The bug is in function naming, not in candidate generation, and it caps this slice for the existing hunter too.",
      "round_zero_detection": null,
      "verdict": "blocked_by_a_naming_gap"
    }
  },
  "development_slice": {
    "vibe-py": {
      "ceiling": 1.0,
      "headroom": 0.167,
      "round_zero_detection": 0.833
    }
  },
  "gaps": [
    [
      "agent-vfc",
      "edutrace-page-ee7ba2",
      "no_candidate"
    ],
    [
      "agent-vfc",
      "lensops-kubernetesmanagement-fe4910",
      "no_candidate"
    ],
    [
      "agent-vfc",
      "money-track-category-icons-4d4a5b",
      "no_candidate"
    ],
    [
      "vfc",
      "chromium-cve-2019-5786",
      "no_candidate"
    ],
    [
      "vfc",
      "chromium-cve-2022-2162",
      "no_candidate"
    ],
    [
      "vfc",
      "curl-cve-2020-8284",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2020-8286",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2021-22890",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2021-22901-ossl_connect_step1",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2022-27775",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2022-27776",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2022-27778",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2022-27782-curl_init_userdefined",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2022-27782-nss_setup_connect",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2022-27782-ossl_connect_step1",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2023-38545",
      "unsupported_language"
    ],
    [
      "vfc",
      "curl-cve-2023-38545-do_socks5",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2021-3449-init_sig_algs_cert",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2021-3712-ia5ncasecmp",
      "no_candidate"
    ],
    [
      "vfc",
      "openssl-cve-2022-0778",
      "no_candidate"
    ],
    [
      "vfc",
      "openssl-cve-2022-1292",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2022-2068-hash_dir",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2022-3358",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2022-3358-evp_cipher_init_internal",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2023-0216",
      "no_candidate"
    ],
    [
      "vfc",
      "openssl-cve-2023-5363",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2024-2511",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2025-69421",
      "no_candidate"
    ],
    [
      "vfc",
      "openssl-cve-2026-14456-expect_quic_for_value",
      "unsupported_language"
    ],
    [
      "vfc",
      "openssl-cve-2026-31790-rsasve_generate",
      "unsupported_language"
    ],
    [
      "vfc-js",
      "apex-publish-static-files-2-0-0",
      "no_candidate"
    ],
    [
      "vfc-js",
      "bestzip-2-1-6",
      "no_candidate"
    ],
    [
      "vfc-js",
      "clamscan-1-2-0",
      "no_candidate"
    ],
    [
      "vfc-js",
      "hot-formula-parser-3-0-0",
      "no_candidate"
    ],
    [
      "vfc-js",
      "pdf-image-1-0-5",
      "no_candidate"
    ],
    [
      "vfc-js",
      "pidusage-1-0-0",
      "no_candidate"
    ],
    [
      "vfc-js",
      "portprocesses-1-0-4",
      "no_candidate"
    ],
    [
      "vfc-js",
      "serve-here-js-1-1-0",
      "no_candidate"
    ]
  ],
  "generated": "2026-09-06",
  "generator": "ir",
  "note": "constrained-detector task 1.3, the stop-or-go reading. A candidate-driven detector can only judge sites the enumerator produces, so this is the limit on its recall, measured offline before any model was called. The condition for continuing: on every deciding slice the ceiling must exceed that slice's recorded detection rate from round zero. It does, with the largest margin exactly where it matters most.",
  "per_family": {
    "access_control": {
      "ceiling": 0.9,
      "pairs": 10,
      "sites": {
        "max": 19,
        "median": 12,
        "min": 4,
        "p90": 19
      },
      "with_candidate": 9
    },
    "config_secrets": {
      "ceiling": 1.0,
      "pairs": 11,
      "sites": {
        "max": 60,
        "median": 15,
        "min": 2,
        "p90": 40
      },
      "with_candidate": 11
    },
    "deserialization": {
      "ceiling": 1.0,
      "pairs": 4,
      "sites": {
        "max": 10,
        "median": 7,
        "min": 3,
        "p90": 10
      },
      "with_candidate": 4
    },
    "injection": {
      "ceiling": 0.8461538461538461,
      "pairs": 52,
      "sites": {
        "max": 77,
        "median": 5,
        "min": 1,
        "p90": 17
      },
      "with_candidate": 44
    },
    "memory": {
      "ceiling": 0.8636363636363636,
      "pairs": 176,
      "sites": {
        "max": 77,
        "median": 10,
        "min": 1,
        "p90": 29
      },
      "with_candidate": 152
    },
    "output_encoding": {
      "ceiling": 1.0,
      "pairs": 2,
      "sites": {
        "max": 17,
        "median": 17,
        "min": 3,
        "p90": 17
      },
      "with_candidate": 2
    },
    "path": {
      "ceiling": 0.875,
      "pairs": 8,
      "sites": {
        "max": 12,
        "median": 7,
        "min": 5,
        "p90": 12
      },
      "with_candidate": 7
    },
    "prototype": {
      "ceiling": 0.7142857142857143,
      "pairs": 7,
      "sites": {
        "max": 14,
        "median": 6,
        "min": 6,
        "p90": 14
      },
      "with_candidate": 5
    },
    "unknown": {
      "ceiling": 0.0,
      "pairs": 2,
      "sites": {
        "max": 0,
        "median": 0,
        "min": 0,
        "p90": 0
      },
      "with_candidate": 0
    },
    "untrusted_destination": {
      "ceiling": 1.0,
      "pairs": 5,
      "sites": {
        "max": 17,
        "median": 7,
        "min": 3,
        "p90": 17
      },
      "with_candidate": 5
    }
  },
  "per_slice": {
    "agent-vfc": {
      "ceiling": 0.896551724137931,
      "pairs": 29,
      "with_candidate": 26
    },
    "github": {
      "ceiling": 1.0,
      "pairs": 6,
      "with_candidate": 6
    },
    "local": {
      "ceiling": 1.0,
      "pairs": 3,
      "with_candidate": 3
    },
    "sast": {
      "ceiling": 1.0,
      "pairs": 11,
      "with_candidate": 11
    },
    "vfc": {
      "ceiling": 0.8465909090909091,
      "pairs": 176,
      "with_candidate": 149
    },
    "vfc-js": {
      "ceiling": 0.5294117647058824,
      "pairs": 17,
      "with_candidate": 9
    },
    "vibe-py": {
      "ceiling": 1.0,
      "pairs": 35,
      "with_candidate": 35
    }
  },
  "shape_correction": "config_secrets first measured 63.6% with a bind-and-literal shape. A permissive default is as often an argument to a call -- cors({origin: '*'}) -- as a binding, and adding call sites to that family's shape took it to 100%. Found by this measurement, before the detector existed.",
  "verdict": "go"
}