Open_Ultra_SAST/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json

{
  "caveat": "30 pairs of ONE slice, and the slice iterated against all session. The +16% overfitting gap against vfc-js is why contributor-scan Req 8.1 and 8.2 exist. A contributor-facing tool has to hold up on a repository nobody tuned against, which is what that spec's Req 4 and 10 are for.",
  "cost_usd": 0.0068,
  "detection_rate": 0.9667,
  "generated": "2026-09-08",
  "hunter_baseline": {
    "leak_rate": 0.123,
    "note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json",
    "pair_correct": 0.767
  },
  "leak_rate": 0.0667,
  "model": "deepseek-v4-flash",
  "outcomes": {
    "both_flagged": 2,
    "both_silent": 1,
    "pair_correct": 27
  },
  "pair_correct_rate": 0.9,
  "pairs": 30,
  "reading": "The wired pipeline beats the LLM hunter on both axes: 90.0% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~100x cheaper ($0.0068 against ~$0.69). 27 of 30 pairs correct, one miss remaining. The earlier 'detection has regressed' reading was taken with the proposer disconnected -- an arbiter judged against a detector -- and before five wiring bugs were found.",
  "remaining_miss": "dsvpwa-dsvpwa-main: its setting is an argparse default (--ssl defaulting to 0), a CLI default rather than a security-setting call. Predicted out of reach before the settings work and still is; contorting the arbiter around one pair would be worse than the miss.",
  "rows": [
    {
      "family": "injection",
      "fixed_findings": 2,
      "outcome": "both_flagged",
      "pair": "damn-vulnerable-graphql-application-views-mutate",
      "vuln_findings": 3,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "damn-vulnerable-graphql-application-views-resolve-pastes",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "damn-vulnerable-graphql-application-views-resolve-system-dia",
      "vuln_findings": 3,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "djangoat-analytics-hits-by-ip",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "deserialization",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "djangoat-views-reset-password",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "djangoat-views-user-view",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 7,
      "outcome": "both_flagged",
      "pair": "dsvpwa-attacks-run",
      "vuln_findings": 7,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "config_secrets",
      "fixed_findings": 0,
      "outcome": "both_silent",
      "pair": "dsvpwa-dsvpwa-main",
      "vuln_findings": 0,
      "vuln_rungs": []
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "dsvpwa-handlers-do-bdr",
      "vuln_findings": 3,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "config_secrets",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "dvpwa-app-init",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "config_secrets",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "dvpwa-middlewares-session-middleware",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "dvpwa-student-create",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "deserialization",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "extremely-vulnerable-flask-app-account-before-request",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "extremely-vulnerable-flask-app-account-search",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "extremely-vulnerable-flask-app-signup-validate-token",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "access_control",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "threatbyte-api-v1-delete",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_corroborated"
      ]
    },
    {
      "family": "access_control",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "threatbyte-api-v1-get",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_corroborated"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "threatbyte-routes-dashboard",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vampi-user-model-get-user",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "access_control",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vampi-users-update-password",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_corroborated"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vfapi-main-delete-user",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vfapi-main-put-user",
      "vuln_findings": 5,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vfapi-main-sql-return-users-from-username",
      "vuln_findings": 1,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "output_encoding",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnerable-python-apps-bad-mvc-do-get",
      "vuln_findings": 5,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "path",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnerable-python-apps-dir-traversal-do-get",
      "vuln_findings": 3,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnpy-cmdi-do-os-system",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnpy-cmdi-do-subprocess-popen",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulpy-libuser-create",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulpy-libuser-login",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulpy-libuser-password-change",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    }
  ],
  "run_history": [
    {
      "note": "absence and config families had no pipeline path to their arbiters",
      "pair_correct": 0.733,
      "run": 1
    },
    {
      "note": "routing fixed -- IDENTICAL result, which was the clue: the arbiter was reached and its verdict discarded",
      "pair_correct": 0.733,
      "run": 2
    },
    {
      "note": "residual question phrased for the arbiter that raised it",
      "pair_correct": 0.8,
      "run": 3
    },
    {
      "note": "aiohttp Application(debug=True) and RedisStorage(httponly=False) added to the settings table",
      "pair_correct": 0.867,
      "run": 4
    },
    {
      "note": "the dominance question no longer contradicts its own evidence line",
      "pair_correct": 0.9,
      "run": 5
    }
  ],
  "seconds": 1213.5,
  "slice": "vibe-py"
}