Open_Ultra_SAST/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json

{
  "cost_note": "Three full runs of this measurement cost about $0.03 in total.",
  "cost_usd": 0.0076,
  "detection_rate": 0.8667,
  "division_of_labour": {
    "model_corroborated": 2,
    "model_entailed": 9,
    "note": "The model arbitrates 11 of 26 findings. Every conversion from suspicion to a model rung removes a leak and a model call at once, which is the payoff curve for sink coverage and interprocedural resolution: closing them moves findings out of the LLM's column into the model's.",
    "suspicion": 15
  },
  "generated": "2026-09-08",
  "hunter_baseline": {
    "leak_rate": 0.123,
    "note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json",
    "pair_correct": 0.767
  },
  "leak_rate": 0.0667,
  "model": "deepseek-v4-flash",
  "outcomes": {
    "both_flagged": 2,
    "both_silent": 4,
    "pair_correct": 24
  },
  "pair_correct_rate": 0.8,
  "pairs": 30,
  "reading": "The wired pipeline beats the LLM hunter on both axes: 80.0% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~90x cheaper ($0.0076 against ~$0.69). This is the first like-for-like comparison; the earlier 'detection has regressed' reading was taken with the proposer disconnected (an arbiter judged against a detector) and before four wiring bugs were found.",
  "rows": [
    {
      "family": "injection",
      "fixed_findings": 2,
      "outcome": "both_flagged",
      "pair": "damn-vulnerable-graphql-application-views-mutate",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "damn-vulnerable-graphql-application-views-resolve-pastes",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "damn-vulnerable-graphql-application-views-resolve-system-dia",
      "vuln_findings": 3,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "djangoat-analytics-hits-by-ip",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "deserialization",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "djangoat-views-reset-password",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "djangoat-views-user-view",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 7,
      "outcome": "both_flagged",
      "pair": "dsvpwa-attacks-run",
      "vuln_findings": 7,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "config_secrets",
      "fixed_findings": 0,
      "outcome": "both_silent",
      "pair": "dsvpwa-dsvpwa-main",
      "vuln_findings": 0,
      "vuln_rungs": []
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "dsvpwa-handlers-do-bdr",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "config_secrets",
      "fixed_findings": 0,
      "outcome": "both_silent",
      "pair": "dvpwa-app-init",
      "vuln_findings": 0,
      "vuln_rungs": []
    },
    {
      "family": "config_secrets",
      "fixed_findings": 0,
      "outcome": "both_silent",
      "pair": "dvpwa-middlewares-session-middleware",
      "vuln_findings": 0,
      "vuln_rungs": []
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "dvpwa-student-create",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "deserialization",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "extremely-vulnerable-flask-app-account-before-request",
      "vuln_findings": 2,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "extremely-vulnerable-flask-app-account-search",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "extremely-vulnerable-flask-app-signup-validate-token",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "access_control",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "threatbyte-api-v1-delete",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_corroborated"
      ]
    },
    {
      "family": "access_control",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "threatbyte-api-v1-get",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_corroborated"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "threatbyte-routes-dashboard",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vampi-user-model-get-user",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "access_control",
      "fixed_findings": 0,
      "outcome": "both_silent",
      "pair": "vampi-users-update-password",
      "vuln_findings": 0,
      "vuln_rungs": []
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vfapi-main-delete-user",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vfapi-main-put-user",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vfapi-main-sql-return-users-from-username",
      "vuln_findings": 1,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "output_encoding",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnerable-python-apps-bad-mvc-do-get",
      "vuln_findings": 4,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "path",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnerable-python-apps-dir-traversal-do-get",
      "vuln_findings": 3,
      "vuln_rungs": [
        "suspicion"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnpy-cmdi-do-os-system",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulnpy-cmdi-do-subprocess-popen",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulpy-libuser-create",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulpy-libuser-login",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    },
    {
      "family": "injection",
      "fixed_findings": 0,
      "outcome": "pair_correct",
      "pair": "vulpy-libuser-password-change",
      "vuln_findings": 1,
      "vuln_rungs": [
        "model_entailed"
      ]
    }
  ],
  "run_history": [
    {
      "leak": 0.067,
      "note": "access_control and config_secrets had no pipeline path to their arbiters",
      "pair_correct": 0.733,
      "run": 1
    },
    {
      "leak": 0.067,
      "note": "family routing fixed -- IDENTICAL result, which was the clue: the arbiter was reached and its verdict discarded",
      "pair_correct": 0.733,
      "run": 2
    },
    {
      "leak": 0.067,
      "note": "residual question phrased for the arbiter that raised it; 2 of 3 access_control pairs converted from silent to pair-correct",
      "pair_correct": 0.8,
      "run": 3
    }
  ],
  "seconds": 1071.1,
  "slice": "vibe-py",
  "still_silent": {
    "access_control": {
      "cause": "vampi-users-update-password: the dominance arbiter returns corroborated for it in isolation, so this is a pipeline-level issue still to diagnose.",
      "pairs": 1
    },
    "config_secrets": {
      "cause": "the settings table is Flask/FastAPI-shaped (CORS, set_cookie, app.run); these pairs are aiohttp (session_middleware, setup_redis). Known since group 4.",
      "pairs": 3
    }
  }
}