mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
merge(learn): exp-004 -- inconclusive, pooled top-30 precision 0.90; experiment fill command; registration fix
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,53 @@
|
|||||||
|
"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking?
|
||||||
|
|
||||||
|
Reads the run's arm_outcome rows from the memory store (OUSAST_MEMORY) and the frozen units file; makes no model call.
|
||||||
|
usage: python benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py (from the repository root)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from openultrasast.config import load_dotenv
|
||||||
|
from openultrasast.learn import experiments as ex
|
||||||
|
from openultrasast.plane.memory import open_store
|
||||||
|
|
||||||
|
EXPERIMENT = "exp-004-pooled-block-gate"
|
||||||
|
UNITS = Path(f"plane/experiments/{EXPERIMENT}.units.jsonl")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
load_dotenv(Path(".env"))
|
||||||
|
rows = [r.row for r in open_store(None).rows(repo=ex.experiment_repo(EXPERIMENT), kind="arm_outcome")]
|
||||||
|
with UNITS.open() as handle:
|
||||||
|
units = {u["unit"]: u for u in (json.loads(line) for line in handle)}
|
||||||
|
print("arm_outcome rows", len(rows), "units file", len(units))
|
||||||
|
by_family: dict[str, list[tuple[float | None, int, str | None, str]]] = collections.defaultdict(list)
|
||||||
|
for row in rows:
|
||||||
|
unit = units.get(row.get("unit"))
|
||||||
|
if unit:
|
||||||
|
by_family[unit["family"]].append((row.get("s"), unit["label"], unit.get("pair"), str(row.get("verdict") or "")))
|
||||||
|
for family, xs in sorted(by_family.items()):
|
||||||
|
scored = [x for x in xs if x[0] is not None]
|
||||||
|
pairs: dict[str, dict[int, float]] = collections.defaultdict(dict)
|
||||||
|
for score, label, pair, _ in scored:
|
||||||
|
if pair:
|
||||||
|
pairs[pair][label] = score
|
||||||
|
full = [p for p in pairs.values() if 0 in p and 1 in p]
|
||||||
|
wins = sum(1 for p in full if p[1] > p[0])
|
||||||
|
ties = sum(1 for p in full if p[1] == p[0])
|
||||||
|
top = sorted((x for x in scored if x[2]), key=lambda x: -(x[0] or 0.0))
|
||||||
|
first_negative = next((i for i, x in enumerate(top) if x[1] == 0), len(top))
|
||||||
|
print(
|
||||||
|
f"{family}: n={len(xs)} distinct_scores={len({round(x[0] or 0.0, 4) for x in scored})} "
|
||||||
|
f"unsure={sum(1 for x in xs if x[3] == 'unsure')} pairs={len(full)} within_pair_win={wins} tie={ties} "
|
||||||
|
f"pos_above_first_neg={first_negative}"
|
||||||
|
)
|
||||||
|
pooled = sorted((x for xs in by_family.values() for x in xs if x[0] is not None and x[2]), key=lambda x: -(x[0] or 0.0))
|
||||||
|
print("pooled raw top-30 labels:", "".join(str(x[1]) for x in pooled[:30]))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
{
|
||||||
|
"experiment": "exp-004-pooled-block-gate",
|
||||||
|
"date": "2026-10-03",
|
||||||
|
"fill": {
|
||||||
|
"client_calls": 2073,
|
||||||
|
"usd": 1.238223,
|
||||||
|
"ceiling_usd": 2.0,
|
||||||
|
"still_missing": 0,
|
||||||
|
"dry_run_estimate_usd": 5.582,
|
||||||
|
"note": "the dry-run estimator over-priced 4.5x (no prompt-cache pricing); measured spend is the meter's"
|
||||||
|
},
|
||||||
|
"run": {
|
||||||
|
"units": 501,
|
||||||
|
"client_calls": 0,
|
||||||
|
"usd": 0.0,
|
||||||
|
"replayed": 2509,
|
||||||
|
"replay_miss": 0
|
||||||
|
},
|
||||||
|
"verdict": "inconclusive",
|
||||||
|
"reason": "no t_block on any outer fold, pooled or per family; B offers no BLOCK",
|
||||||
|
"instrument_check": {
|
||||||
|
"per_family": {
|
||||||
|
"access_control": {
|
||||||
|
"n": 66,
|
||||||
|
"distinct_scores": 41,
|
||||||
|
"unsure": 5,
|
||||||
|
"pairs": 28,
|
||||||
|
"within_pair_wins": 22,
|
||||||
|
"ties": 1,
|
||||||
|
"positives_above_first_negative": 12
|
||||||
|
},
|
||||||
|
"deserialization": {
|
||||||
|
"n": 64,
|
||||||
|
"distinct_scores": 38,
|
||||||
|
"unsure": 7,
|
||||||
|
"pairs": 25,
|
||||||
|
"within_pair_wins": 18,
|
||||||
|
"ties": 4,
|
||||||
|
"positives_above_first_negative": 5
|
||||||
|
},
|
||||||
|
"injection": {
|
||||||
|
"n": 149,
|
||||||
|
"distinct_scores": 52,
|
||||||
|
"unsure": 54,
|
||||||
|
"pairs": 53,
|
||||||
|
"within_pair_wins": 30,
|
||||||
|
"ties": 17,
|
||||||
|
"positives_above_first_negative": 6
|
||||||
|
},
|
||||||
|
"output_encoding": {
|
||||||
|
"n": 69,
|
||||||
|
"distinct_scores": 31,
|
||||||
|
"unsure": 32,
|
||||||
|
"pairs": 28,
|
||||||
|
"within_pair_wins": 18,
|
||||||
|
"ties": 6,
|
||||||
|
"positives_above_first_negative": 2
|
||||||
|
},
|
||||||
|
"path": {
|
||||||
|
"n": 66,
|
||||||
|
"distinct_scores": 43,
|
||||||
|
"unsure": 15,
|
||||||
|
"pairs": 29,
|
||||||
|
"within_pair_wins": 25,
|
||||||
|
"ties": 3,
|
||||||
|
"positives_above_first_negative": 13
|
||||||
|
},
|
||||||
|
"untrusted_destination": {
|
||||||
|
"n": 87,
|
||||||
|
"distinct_scores": 36,
|
||||||
|
"unsure": 33,
|
||||||
|
"pairs": 32,
|
||||||
|
"within_pair_wins": 18,
|
||||||
|
"ties": 4,
|
||||||
|
"positives_above_first_negative": 3
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pooled_raw_top30_labels": "111111111110111110111111110111",
|
||||||
|
"reading": "scores are real (31-52 distinct values per family); pooled raw top 30 holds 27 positives (0.90), first negative at rank 12; the Wilson bound needs >= 73 flags with no error, so precision at the top, not data volume alone, now binds"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,822 @@
|
|||||||
|
{
|
||||||
|
"arms": {
|
||||||
|
"A": {
|
||||||
|
"label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p",
|
||||||
|
"rule": {
|
||||||
|
"name": "calibrated_block"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"label": "pooled gate -- one t_block over all families on a per-family Platt score (training folds, inner cross-fit), t_quiet per family at 90% recall, family floor 10 clean held-out flags",
|
||||||
|
"rule": {
|
||||||
|
"floor": 10,
|
||||||
|
"name": "pooled_block_gate",
|
||||||
|
"normalise": "platt",
|
||||||
|
"precision": 0.95,
|
||||||
|
"recall": 0.9,
|
||||||
|
"select_on": "paired"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"code_commit": "90a2642019df414cb67b5dcea6e4ec235f35650a",
|
||||||
|
"decision": "inconclusive",
|
||||||
|
"experiment": "exp-004-pooled-block-gate",
|
||||||
|
"families": {
|
||||||
|
"access_control": {
|
||||||
|
"A": {
|
||||||
|
"calibration_holds": false,
|
||||||
|
"ece": 0.0715,
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 28,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 28,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 56
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 37,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 38,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 66
|
||||||
|
},
|
||||||
|
"reason_by_fold": {
|
||||||
|
"outer-0": "calibration",
|
||||||
|
"outer-1": "calibration",
|
||||||
|
"outer-2": "calibration",
|
||||||
|
"outer-3": "calibration",
|
||||||
|
"outer-4": "calibration"
|
||||||
|
},
|
||||||
|
"thresholds_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"floor": {
|
||||||
|
"errors": 0,
|
||||||
|
"flagged": 0,
|
||||||
|
"floor": 10,
|
||||||
|
"may_block": false
|
||||||
|
},
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 28,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 28,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 56
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 37,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 38,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 66
|
||||||
|
},
|
||||||
|
"quiet_recall": {
|
||||||
|
"paired": 0.8929,
|
||||||
|
"pooled": 0.9211
|
||||||
|
},
|
||||||
|
"t_block_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
},
|
||||||
|
"t_quiet_by_fold": {
|
||||||
|
"outer-0": 0.44900641988143913,
|
||||||
|
"outer-1": 0.3723384028044276,
|
||||||
|
"outer-2": 0.4202772526614184,
|
||||||
|
"outer-3": 0.4481849709797749,
|
||||||
|
"outer-4": 0.36346174463707154
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capacity": {
|
||||||
|
"evaluated": 66,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_positives_by_training_folds": {
|
||||||
|
"outer-0": 22,
|
||||||
|
"outer-1": 22,
|
||||||
|
"outer-2": 22,
|
||||||
|
"outer-3": 23,
|
||||||
|
"outer-4": 23
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"deserialization": {
|
||||||
|
"A": {
|
||||||
|
"calibration_holds": false,
|
||||||
|
"ece": 0.0725,
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 25,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 25,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 50
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 39,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 39,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 64
|
||||||
|
},
|
||||||
|
"reason_by_fold": {
|
||||||
|
"outer-0": "calibration",
|
||||||
|
"outer-1": "calibration",
|
||||||
|
"outer-2": "calibration",
|
||||||
|
"outer-3": "calibration",
|
||||||
|
"outer-4": "calibration"
|
||||||
|
},
|
||||||
|
"thresholds_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"floor": {
|
||||||
|
"errors": 0,
|
||||||
|
"flagged": 0,
|
||||||
|
"floor": 10,
|
||||||
|
"may_block": false
|
||||||
|
},
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 25,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 25,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 50
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 39,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 39,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 64
|
||||||
|
},
|
||||||
|
"quiet_recall": {
|
||||||
|
"paired": 0.88,
|
||||||
|
"pooled": 0.9231
|
||||||
|
},
|
||||||
|
"t_block_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
},
|
||||||
|
"t_quiet_by_fold": {
|
||||||
|
"outer-0": 0.17507194490500488,
|
||||||
|
"outer-1": 0.22766488322270723,
|
||||||
|
"outer-2": 0.2163745870743959,
|
||||||
|
"outer-3": 0.3112201878893598,
|
||||||
|
"outer-4": 0.363243722721776
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capacity": {
|
||||||
|
"evaluated": 64,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_positives_by_training_folds": {
|
||||||
|
"outer-0": 20,
|
||||||
|
"outer-1": 20,
|
||||||
|
"outer-2": 20,
|
||||||
|
"outer-3": 20,
|
||||||
|
"outer-4": 20
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"injection": {
|
||||||
|
"A": {
|
||||||
|
"calibration_holds": false,
|
||||||
|
"ece": 0.1042,
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 41,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 53,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 106
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 49,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 78,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 149
|
||||||
|
},
|
||||||
|
"reason_by_fold": {
|
||||||
|
"outer-0": "calibration",
|
||||||
|
"outer-1": "calibration",
|
||||||
|
"outer-2": "calibration",
|
||||||
|
"outer-3": "calibration",
|
||||||
|
"outer-4": "calibration"
|
||||||
|
},
|
||||||
|
"thresholds_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"floor": {
|
||||||
|
"errors": 0,
|
||||||
|
"flagged": 0,
|
||||||
|
"floor": 10,
|
||||||
|
"may_block": false
|
||||||
|
},
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 41,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 53,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 106
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 49,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 78,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 149
|
||||||
|
},
|
||||||
|
"quiet_recall": {
|
||||||
|
"paired": 0.8679,
|
||||||
|
"pooled": 0.8974
|
||||||
|
},
|
||||||
|
"t_block_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
},
|
||||||
|
"t_quiet_by_fold": {
|
||||||
|
"outer-0": 0.287983504097999,
|
||||||
|
"outer-1": 0.17524866355153343,
|
||||||
|
"outer-2": 0.12918182665453967,
|
||||||
|
"outer-3": 0.14521040470628938,
|
||||||
|
"outer-4": 0.23005714687697673
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capacity": {
|
||||||
|
"evaluated": 149,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_positives_by_training_folds": {
|
||||||
|
"outer-0": 38,
|
||||||
|
"outer-1": 44,
|
||||||
|
"outer-2": 45,
|
||||||
|
"outer-3": 43,
|
||||||
|
"outer-4": 42
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"output_encoding": {
|
||||||
|
"A": {
|
||||||
|
"calibration_holds": false,
|
||||||
|
"ece": 0.0736,
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 28,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 28,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 56
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 39,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 41,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 69
|
||||||
|
},
|
||||||
|
"reason_by_fold": {
|
||||||
|
"outer-0": "calibration",
|
||||||
|
"outer-1": "calibration",
|
||||||
|
"outer-2": "calibration",
|
||||||
|
"outer-3": "calibration",
|
||||||
|
"outer-4": "calibration"
|
||||||
|
},
|
||||||
|
"thresholds_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"floor": {
|
||||||
|
"errors": 0,
|
||||||
|
"flagged": 0,
|
||||||
|
"floor": 10,
|
||||||
|
"may_block": false
|
||||||
|
},
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 28,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 28,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 56
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 39,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 41,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 69
|
||||||
|
},
|
||||||
|
"quiet_recall": {
|
||||||
|
"paired": 0.8571,
|
||||||
|
"pooled": 0.9024
|
||||||
|
},
|
||||||
|
"t_block_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
},
|
||||||
|
"t_quiet_by_fold": {
|
||||||
|
"outer-0": 0.3723166587726901,
|
||||||
|
"outer-1": 0.5329072757249385,
|
||||||
|
"outer-2": 0.3785066578507705,
|
||||||
|
"outer-3": 0.4389011844848602,
|
||||||
|
"outer-4": 0.43855966643440264
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capacity": {
|
||||||
|
"evaluated": 69,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_positives_by_training_folds": {
|
||||||
|
"outer-0": 23,
|
||||||
|
"outer-1": 22,
|
||||||
|
"outer-2": 22,
|
||||||
|
"outer-3": 22,
|
||||||
|
"outer-4": 23
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"path": {
|
||||||
|
"A": {
|
||||||
|
"calibration_holds": false,
|
||||||
|
"ece": 0.0621,
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 29,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 29,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 58
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 37,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 37,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 66
|
||||||
|
},
|
||||||
|
"reason_by_fold": {
|
||||||
|
"outer-0": "calibration",
|
||||||
|
"outer-1": "calibration",
|
||||||
|
"outer-2": "calibration",
|
||||||
|
"outer-3": "calibration",
|
||||||
|
"outer-4": "calibration"
|
||||||
|
},
|
||||||
|
"thresholds_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"floor": {
|
||||||
|
"errors": 0,
|
||||||
|
"flagged": 0,
|
||||||
|
"floor": 10,
|
||||||
|
"may_block": false
|
||||||
|
},
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 29,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 29,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 58
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 37,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 37,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 66
|
||||||
|
},
|
||||||
|
"quiet_recall": {
|
||||||
|
"paired": 0.8621,
|
||||||
|
"pooled": 0.8919
|
||||||
|
},
|
||||||
|
"t_block_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
},
|
||||||
|
"t_quiet_by_fold": {
|
||||||
|
"outer-0": 0.4680507207548809,
|
||||||
|
"outer-1": 0.3955717127313657,
|
||||||
|
"outer-2": 0.43694703358760895,
|
||||||
|
"outer-3": 0.4450583743716199,
|
||||||
|
"outer-4": 0.46396657833309796
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capacity": {
|
||||||
|
"evaluated": 66,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_positives_by_training_folds": {
|
||||||
|
"outer-0": 23,
|
||||||
|
"outer-1": 23,
|
||||||
|
"outer-2": 24,
|
||||||
|
"outer-3": 23,
|
||||||
|
"outer-4": 23
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"untrusted_destination": {
|
||||||
|
"A": {
|
||||||
|
"calibration_holds": false,
|
||||||
|
"ece": 0.1143,
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 31,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 32,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 64
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 49,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 50,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 87
|
||||||
|
},
|
||||||
|
"reason_by_fold": {
|
||||||
|
"outer-0": "calibration",
|
||||||
|
"outer-1": "calibration",
|
||||||
|
"outer-2": "calibration",
|
||||||
|
"outer-3": "calibration",
|
||||||
|
"outer-4": "calibration"
|
||||||
|
},
|
||||||
|
"thresholds_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"floor": {
|
||||||
|
"errors": 0,
|
||||||
|
"flagged": 0,
|
||||||
|
"floor": 10,
|
||||||
|
"may_block": false
|
||||||
|
},
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 31,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 32,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 64
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 49,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 50,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 87
|
||||||
|
},
|
||||||
|
"quiet_recall": {
|
||||||
|
"paired": 1.0,
|
||||||
|
"pooled": 1.0
|
||||||
|
},
|
||||||
|
"t_block_by_fold": {
|
||||||
|
"outer-0": null,
|
||||||
|
"outer-1": null,
|
||||||
|
"outer-2": null,
|
||||||
|
"outer-3": null,
|
||||||
|
"outer-4": null
|
||||||
|
},
|
||||||
|
"t_quiet_by_fold": {
|
||||||
|
"outer-0": 0.13676746287025585,
|
||||||
|
"outer-1": 0.11255775703980551,
|
||||||
|
"outer-2": 0.13763698150768486,
|
||||||
|
"outer-3": 0.09787938647374639,
|
||||||
|
"outer-4": 0.13763698150768486
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capacity": {
|
||||||
|
"evaluated": 87,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_positives_by_training_folds": {
|
||||||
|
"outer-0": 26,
|
||||||
|
"outer-1": 26,
|
||||||
|
"outer-2": 24,
|
||||||
|
"outer-3": 26,
|
||||||
|
"outer-4": 26
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"hypothesis": "One BLOCK threshold pooled across families on a per-family Platt-normalised score, chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95, reaches that bound on held-out repositories' paired units, where per-family thresholds (exp-003) could not for lack of flags; a per-family floor (>= 10 held-out flags, none wrong) keeps a family that the pooling masks from blocking.\n",
|
||||||
|
"manifest_digest": "87fceb6580c6de629d6a6089eb5888cdfc4df0f791576184f987c14c192d85f5",
|
||||||
|
"min_flagged_for_bound": 73,
|
||||||
|
"overall": {
|
||||||
|
"A": {
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 182,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 195,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 390
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 220,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 283,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 501
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 182,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 195,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 390
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 220,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 283,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 501
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overall_after_floor": {
|
||||||
|
"B": {
|
||||||
|
"paired": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 182,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 195,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 390
|
||||||
|
},
|
||||||
|
"pooled": {
|
||||||
|
"coverage": 0.0,
|
||||||
|
"flagged": 0,
|
||||||
|
"flagged_precision": null,
|
||||||
|
"flagged_precision_ci": null,
|
||||||
|
"flagged_precision_lb": null,
|
||||||
|
"groups": 220,
|
||||||
|
"meets_target_lb": false,
|
||||||
|
"meets_target_point": false,
|
||||||
|
"positives": 283,
|
||||||
|
"true_flagged": 0,
|
||||||
|
"units": 501
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"populations": {
|
||||||
|
"primary": "paired",
|
||||||
|
"secondary": [
|
||||||
|
"pooled"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"primary": [
|
||||||
|
{
|
||||||
|
"name": "flagged_precision_lb",
|
||||||
|
"target": 0.95
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"reason": "B offers no BLOCK on held-out paired units; the incumbent stays",
|
||||||
|
"resamples": 2000,
|
||||||
|
"secondary": [
|
||||||
|
{
|
||||||
|
"name": "flagged_precision"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "coverage"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "flagged"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "quiet_recall"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"spend_usd": {
|
||||||
|
"score": 0.0
|
||||||
|
},
|
||||||
|
"target_precision": 0.95,
|
||||||
|
"units": {
|
||||||
|
"by_label": {
|
||||||
|
"0": 218,
|
||||||
|
"1": 283
|
||||||
|
},
|
||||||
|
"evaluated": 501,
|
||||||
|
"excluded_replay_miss": 0,
|
||||||
|
"frozen": 501,
|
||||||
|
"groups": 220,
|
||||||
|
"not_run": 0,
|
||||||
|
"paired_evaluated": 390,
|
||||||
|
"unsure": 146
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"modules": 157,
|
"modules": 158,
|
||||||
"counts": {
|
"counts": {
|
||||||
"load_bearing": 152,
|
"load_bearing": 153,
|
||||||
"standalone_capability": 5
|
"standalone_capability": 5
|
||||||
},
|
},
|
||||||
"orphaned": [],
|
"orphaned": [],
|
||||||
@@ -358,6 +358,7 @@
|
|||||||
"learn.excerpt",
|
"learn.excerpt",
|
||||||
"learn.experiments",
|
"learn.experiments",
|
||||||
"learn.features",
|
"learn.features",
|
||||||
|
"learn.fill",
|
||||||
"learn.folds",
|
"learn.folds",
|
||||||
"learn.labels",
|
"learn.labels",
|
||||||
"learn.leaks",
|
"learn.leaks",
|
||||||
@@ -412,7 +413,8 @@
|
|||||||
"importers": [
|
"importers": [
|
||||||
"learn.cli",
|
"learn.cli",
|
||||||
"learn.compile",
|
"learn.compile",
|
||||||
"learn.experiments"
|
"learn.experiments",
|
||||||
|
"learn.fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -424,6 +426,7 @@
|
|||||||
"learn.compile",
|
"learn.compile",
|
||||||
"learn.evaluate",
|
"learn.evaluate",
|
||||||
"learn.experiments",
|
"learn.experiments",
|
||||||
|
"learn.fill",
|
||||||
"learn.folds",
|
"learn.folds",
|
||||||
"learn.program",
|
"learn.program",
|
||||||
"learn.retrieve"
|
"learn.retrieve"
|
||||||
@@ -445,7 +448,8 @@
|
|||||||
"classification": "load_bearing",
|
"classification": "load_bearing",
|
||||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||||
"importers": [
|
"importers": [
|
||||||
"learn.cli"
|
"learn.cli",
|
||||||
|
"learn.fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -457,6 +461,14 @@
|
|||||||
"plane.tasks.features"
|
"plane.tasks.features"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"module": "learn.fill",
|
||||||
|
"classification": "load_bearing",
|
||||||
|
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||||
|
"importers": [
|
||||||
|
"learn.cli"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"module": "learn.folds",
|
"module": "learn.folds",
|
||||||
"classification": "load_bearing",
|
"classification": "load_bearing",
|
||||||
@@ -502,7 +514,8 @@
|
|||||||
"learn.cli",
|
"learn.cli",
|
||||||
"learn.compile",
|
"learn.compile",
|
||||||
"learn.evaluate",
|
"learn.evaluate",
|
||||||
"learn.experiments"
|
"learn.experiments",
|
||||||
|
"learn.fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -585,6 +598,7 @@
|
|||||||
"findings",
|
"findings",
|
||||||
"learn.cli",
|
"learn.cli",
|
||||||
"learn.features",
|
"learn.features",
|
||||||
|
"learn.fill",
|
||||||
"learn.labels",
|
"learn.labels",
|
||||||
"model.candidates",
|
"model.candidates",
|
||||||
"model.config_value",
|
"model.config_value",
|
||||||
@@ -678,6 +692,7 @@
|
|||||||
"importers": [
|
"importers": [
|
||||||
"cli",
|
"cli",
|
||||||
"learn.cli",
|
"learn.cli",
|
||||||
|
"learn.fill",
|
||||||
"plane.budget",
|
"plane.budget",
|
||||||
"plane.generate",
|
"plane.generate",
|
||||||
"push.assistance",
|
"push.assistance",
|
||||||
@@ -854,6 +869,7 @@
|
|||||||
"learn.embeddings",
|
"learn.embeddings",
|
||||||
"learn.examples",
|
"learn.examples",
|
||||||
"learn.experiments",
|
"learn.experiments",
|
||||||
|
"learn.fill",
|
||||||
"learn.program",
|
"learn.program",
|
||||||
"plane.delivery",
|
"plane.delivery",
|
||||||
"plane.doctor",
|
"plane.doctor",
|
||||||
@@ -881,6 +897,7 @@
|
|||||||
"learn.compile",
|
"learn.compile",
|
||||||
"learn.embeddings",
|
"learn.embeddings",
|
||||||
"learn.experiments",
|
"learn.experiments",
|
||||||
|
"learn.fill",
|
||||||
"learn.program",
|
"learn.program",
|
||||||
"plane.generate",
|
"plane.generate",
|
||||||
"plane.tasks.roles",
|
"plane.tasks.roles",
|
||||||
@@ -964,6 +981,7 @@
|
|||||||
"learn.embeddings",
|
"learn.embeddings",
|
||||||
"learn.examples",
|
"learn.examples",
|
||||||
"learn.experiments",
|
"learn.experiments",
|
||||||
|
"learn.fill",
|
||||||
"learn.program",
|
"learn.program",
|
||||||
"plane.delivery",
|
"plane.delivery",
|
||||||
"plane.doctor",
|
"plane.doctor",
|
||||||
@@ -1819,6 +1837,8 @@
|
|||||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||||
"importers": [
|
"importers": [
|
||||||
"cli",
|
"cli",
|
||||||
|
"learn.cli",
|
||||||
|
"learn.fill",
|
||||||
"model.endpoint",
|
"model.endpoint",
|
||||||
"plane.budget",
|
"plane.budget",
|
||||||
"plane.tasks.roles",
|
"plane.tasks.roles",
|
||||||
|
|||||||
+67
-3
@@ -150,7 +150,7 @@ Four experiments are registered:
|
|||||||
0.875 against 0.906 and an interval spanning zero
|
0.875 against 0.906 and an interval spanning zero
|
||||||
(`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`).
|
(`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`).
|
||||||
- exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything.
|
- exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything.
|
||||||
- exp-004 tests one pooled BLOCK threshold with a per-family floor. It is registered and has not run.
|
- exp-004 tested one pooled BLOCK threshold with a per-family floor. It was **inconclusive**.
|
||||||
|
|
||||||
### exp-003: precision-bound blocking (2026-10-03)
|
### exp-003: precision-bound blocking (2026-10-03)
|
||||||
|
|
||||||
@@ -216,8 +216,44 @@ Calibration holds only for untrusted destination.
|
|||||||
There arm A still finds no BLOCK point that reaches the bound.
|
There arm A still finds no BLOCK point that reaches the bound.
|
||||||
Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json`.
|
Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json`.
|
||||||
|
|
||||||
|
### exp-004: pooled block gate (2026-10-03)
|
||||||
|
|
||||||
|
The experiment tested one pooled BLOCK threshold across families.
|
||||||
|
Each family needed at least 10 clean flags to block.
|
||||||
|
Each family also had its own quiet threshold.
|
||||||
|
The run evaluated 501 units across 220 repository groups.
|
||||||
|
Of these, 390 units were paired.
|
||||||
|
The cache fill cost $1.238223 for 2,073 calls.
|
||||||
|
The replay cost $0.0 and made 0 client calls.
|
||||||
|
|
||||||
|
The verdict was **inconclusive**.
|
||||||
|
No fold found a `t_block` threshold.
|
||||||
|
The instrument check confirmed real scores.
|
||||||
|
The pooled top 30 contained 27 positives.
|
||||||
|
The first negative appeared at rank 12.
|
||||||
|
The bound needs 73 clean flags without an error.
|
||||||
|
|
||||||
|
A within-pair win means the vulnerable side scored above its fixed counterpart.
|
||||||
|
|
||||||
|
| Family | Within-pair wins | Pairs |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| access control | 22 | 28 |
|
||||||
|
| deserialization | 18 | 25 |
|
||||||
|
| injection | 30 | 53 |
|
||||||
|
| output encoding | 18 | 28 |
|
||||||
|
| path | 25 | 29 |
|
||||||
|
| untrusted destination | 18 | 32 |
|
||||||
|
|
||||||
|
Precision at the top of the ranking now limits blocking.
|
||||||
|
Data volume alone cannot resolve this.
|
||||||
|
The next levers are graph evidence in the prompt and the combiner.
|
||||||
|
|
||||||
|
Records: `benchmarks/experiments/exp-004-pooled-block-gate/record.json` and `benchmarks/experiments/exp-004-pooled-block-gate/result.json`.
|
||||||
|
|
||||||
### exp-004: pooled block gate with a family floor (registered, not run)
|
### exp-004: pooled block gate with a family floor (registered, not run)
|
||||||
|
|
||||||
|
This registration describes the setup before the run reported above.
|
||||||
|
|
||||||
The maintainer amended the BLOCK rule on 2026-10-03 (requirements, Req 6.4).
|
The maintainer amended the BLOCK rule on 2026-10-03 (requirements, Req 6.4).
|
||||||
Precision is now shown across all families together, not per family.
|
Precision is now shown across all families together, not per family.
|
||||||
Each family's score is first mapped by its own Platt fit, on training folds.
|
Each family's score is first mapped by its own Platt fit, on training folds.
|
||||||
@@ -230,8 +266,8 @@ No held-out label moves a threshold.
|
|||||||
|
|
||||||
- **Primary metric:** the pooled held-out Wilson lower bound of flagged precision, paired units, before the floor.
|
- **Primary metric:** the pooled held-out Wilson lower bound of flagged precision, paired units, before the floor.
|
||||||
- **Decision:** reject below point precision 0.95; adopt if the bound reaches 0.95 and a family passes the floor.
|
- **Decision:** reject below point precision 0.95; adopt if the bound reaches 0.95 and a family passes the floor.
|
||||||
- **Units:** frozen at run time, from the memory after the negatives harvest.
|
- **Units:** frozen from the memory after the negatives harvest.
|
||||||
- **Cost:** the run replays the cache. The new memory needs a paid cache fill first.
|
- **Cost:** the run replayed the cache after a paid cache fill.
|
||||||
|
|
||||||
Manifest: `plane/experiments/exp-004-pooled-block-gate.yaml`.
|
Manifest: `plane/experiments/exp-004-pooled-block-gate.yaml`.
|
||||||
|
|
||||||
@@ -252,6 +288,34 @@ The reading is on [Where we stand](where-we-stand.md#2-our-approach-to-detection
|
|||||||
| `ousast learn curve --program ID` | the metric over memory size on a fixed candidate subset |
|
| `ousast learn curve --program ID` | the metric over memory size on a fixed candidate subset |
|
||||||
| `ousast learn audit-leaks` | how well each signal alone separates the sides of a pair (no model) |
|
| `ousast learn audit-leaks` | how well each signal alone separates the sides of a pair (no model) |
|
||||||
| `ousast learn experiment register\|units\|run\|analyse MANIFEST` | A/B experiments from a pre-registered manifest (`plane/experiments/<id>.yaml`): record its digest, freeze the units (no model), run the arms paired (arm A may replay from the cache at $0), analyse (repository bootstrap, exact McNemar, two looks, the adoption verdict) |
|
| `ousast learn experiment register\|units\|run\|analyse MANIFEST` | A/B experiments from a pre-registered manifest (`plane/experiments/<id>.yaml`): record its digest, freeze the units (no model), run the arms paired (arm A may replay from the cache at $0), analyse (repository bootstrap, exact McNemar, two looks, the adoption verdict) |
|
||||||
|
| `ousast learn experiment fill MANIFEST --budget-usd X` | Fill missing responses for a registered `kind: rule` experiment's frozen units before its zero-cost run; `--dry-run` inspects and estimates without calls or writes |
|
||||||
|
|
||||||
|
`experiment fill` accepts `--memory URL`, `--profile static|plane` (default `static`),
|
||||||
|
`--manifest-model PATH` (default `plane/models/deepseek-flash.yaml`) and `--out PATH`.
|
||||||
|
Freeze, commit and register the units first. Fill uses the incumbent programs and the
|
||||||
|
same retrieval, sampling and cache keys as replay. The first missing request is the
|
||||||
|
smoke call; a successful smoke response is kept. Any provider failure stops the fill.
|
||||||
|
|
||||||
|
The JSON report on stdout gives each family's units, distinct requests, initially
|
||||||
|
cached and missing responses, newly filled responses and still-missing responses,
|
||||||
|
plus total client calls and metered USD. `--out` also saves it, except during dry-run,
|
||||||
|
which writes nothing. Exit codes are 0 for completion or dry-run, 1 for a provider/cache
|
||||||
|
failure, 2 for invalid setup and 3 for budget exhaustion. Resume a partial fill with a
|
||||||
|
new invocation and budget; that budget covers only the new invocation, including smoke.
|
||||||
|
|
||||||
|
Dry-run uses each family's mean billed cached response cost when recorded usage exists.
|
||||||
|
Model prices account for cache-hit input, cache-miss input, and output tokens.
|
||||||
|
Without cached usage, it uses the program's input-token estimate and **300 output tokens**.
|
||||||
|
Each family's `estimate_method` reports `measured` or `estimated`.
|
||||||
|
It reports potential retries separately as `conditional_retries`: an unseen answer
|
||||||
|
may need one parse retry, whose necessity cannot be known without calling the model.
|
||||||
|
Known invalid cached answers include their retry requests in the inventory.
|
||||||
|
|
||||||
|
Fill reserves at least **$0.01** and the largest observed call cost before each call.
|
||||||
|
For the production client it also reserves a conservative input-byte token bound plus
|
||||||
|
the program's enforced output-token limit; hidden provider retries are disabled.
|
||||||
|
Consequently a small budget may stop before making any call, and a fill may finish
|
||||||
|
with unused budget. Calls without token usage stop as failures, never as free calls.
|
||||||
|
|
||||||
The memory store is the plane's (`OUSAST_MEMORY`; see [ops/ax/README.md](ops/ax/README.md)).
|
The memory store is the plane's (`OUSAST_MEMORY`; see [ops/ax/README.md](ops/ax/README.md)).
|
||||||
The harvests that produced the plane signals ran as plane Runs generated by
|
The harvests that produced the plane signals ran as plane Runs generated by
|
||||||
|
|||||||
@@ -0,0 +1,501 @@
|
|||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "adonisjs/http-server", "label": 0, "pair": "6b0b3cab2df3397b", "unit": "0ac37b7154242dfa637140af643fb4595e85510a17ab1a12c111441b9d293238"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "adonisjs/http-server", "label": 1, "pair": "6b0b3cab2df3397b", "unit": "6e5e274946d0e8f13a7ef4e37f6c7ed1828d057304a9262db0311413f18e1731"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "agentfront/frontmcp", "label": 0, "pair": "c0d0c91757f5ba95", "unit": "285891b16ee6870af50b36dec3c2ca9a90ee15d9e14851c61e66ad443e6eab59"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "agentfront/frontmcp", "label": 1, "pair": "c0d0c91757f5ba95", "unit": "f33fb2413edf1499a524c4592b3b2f93d166be845f5b65291969a086a4bec334"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "ahujasid/blender-mcp", "label": 1, "pair": "f56f2a107ae591be", "unit": "350149bf9cafbb547f500b714a164734d2361b22e86fc9819c7bad3507d8458a"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "ahujasid/blender-mcp", "label": 0, "pair": "f56f2a107ae591be", "unit": "ad125319dd2d4c9d2d99a1eb34bb4cfab9f099a96e7547d0d59a8ecf1c990347"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "aio-libs/aiohttp", "label": 1, "pair": "ee577ee4b6076fab", "unit": "2523f55d035d1a772a842090bc335e5d755b2fe15b376dd5b6b9e30b9dae82d1"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "aio-libs/aiohttp", "label": 0, "pair": "ee577ee4b6076fab", "unit": "ca0f6c160d7839204f6fcc1709bcbc749a02a0b3b8bb648e2a238eacbf22ed78"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "alchemycms/alchemy_cms", "label": 1, "pair": "c95b6b01d0c2db2a", "unit": "15540c23be0c42472a49039658ec2ec0bbd8c6b318e35dd040a9d427370e0ef0"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "alchemycms/alchemy_cms", "label": 0, "pair": "c95b6b01d0c2db2a", "unit": "542471980417e964182d7d3db59a05fe12bdd072449f3bd77f7daa5515dbc909"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "angular/angular", "label": 0, "pair": "29252934ef4bcfcf", "unit": "63bae9240b31f9cf1dfe00d9b01d4d0ffda95931c9e7d040df1fd718f04d828b"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "angular/angular", "label": 1, "pair": "29252934ef4bcfcf", "unit": "cce2af1743834dcc3df58827ac263a2d63296ca1b2a2f9bdb811954fb5767c33"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "bcc636b23dfb5f570d8864b17fe07b6b4fb52b65a40988c32011f1422f74d504"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "dbcf1e67a1be7ccf7d51701f9b8339339164055135443044c23a446b16b3d934"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "50408f68114ecb4de3bebd836d52d9afad86b22dd1843641ac8a49b617c09e33"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "anxolerd/dvpwa", "label": 1, "pair": "", "unit": "cafb0f00f80dc6beeba40eb95c764bf9a3569193070ba321c28677dfa425ec88"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "apache/airflow", "label": 0, "pair": "07317a2258292656", "unit": "a9f292d7cb467def582d9c0b31a397b13cdaefa77380ce971bfe9fbf535bb660"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "apache/airflow", "label": 1, "pair": "07317a2258292656", "unit": "ed363d0dfa620707be477b90eb4b83a901bfd402040b918aeee6f06b2c5b795c"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "apache/inlong", "label": 1, "pair": "1544b6dfa2299d60", "unit": "d45018fc4ce3dfd715c7087d4d197ad4108c0c8b6625c02c4d4da0adb9f6a140"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "apache/inlong", "label": 0, "pair": "1544b6dfa2299d60", "unit": "f295c7dfb095595a4aa985c58688e4757c29be9e06b7a4ca55035ac5d06b1ef8"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "apache/logging-log4net", "label": 1, "pair": "fe0e104d111c2555", "unit": "088036fab9962be2b96f9162ea4f32190575678d3c5ba96bdfeac178bc22b2c4"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "apache/logging-log4net", "label": 0, "pair": "fe0e104d111c2555", "unit": "7fd2b56a816e25358477eee64487fb2c77f34c652703ad0a7120c368492017a3"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "apache/wicket", "label": 1, "pair": "b6742570f7f0e026", "unit": "7c6c1d27aee6557f1b6bf371b8209003f0d2d1122cfa35b6629eb43f38769cf4"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "apache/wicket", "label": 0, "pair": "b6742570f7f0e026", "unit": "df8300d84dd784d0be46969ea432ef97b8ce611bae7a896ed4f349bd4d6897db"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "apolloconfig/apollo", "label": 1, "pair": "0f584bb97a154488", "unit": "222a84ffc7c58d45e8f10df221762b23bf20d0287ecd20f375558134928886ca"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "apolloconfig/apollo", "label": 0, "pair": "0f584bb97a154488", "unit": "25e4c9d2941f59f072385a7bde3fd0bd76f9e986ed24b06ce81962ab447f5094"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "appium/java-client", "label": 0, "pair": "b15b924b4f7b1ae1", "unit": "2fa95fce0751c97db1a12a80ddb249ff5c54ee31c7dc3cd47765099c7b673e52"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "appium/java-client", "label": 1, "pair": "b15b924b4f7b1ae1", "unit": "52be8128053102eafbee67c2d3aac88c83ca290b943ab2f2cfdcd2630601c7d9"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "arcadedata/arcadedb", "label": 0, "pair": "d31ca58f2a1d27d1", "unit": "2ae40ea61d691060e7b9f6c8e910e3aa0edf16bc6f88bec81aa9a7dae5cf5603"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "arcadedata/arcadedb", "label": 1, "pair": "d31ca58f2a1d27d1", "unit": "4876728701698045e65eeae6e11e133c30f43b09029d7777d6d780f7d0855885"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "archivy/archivy", "label": 0, "pair": "bb3d7baed62e6bb7", "unit": "e8ae88fd2b677e3f64ac94bddb012e3b903afa294ecc87774d10087c0d50489c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "archivy/archivy", "label": 1, "pair": "bb3d7baed62e6bb7", "unit": "f6b0d241891dc1d897d74d98b6ff7ada2c1f1ce7874ca32097630c06ca53e389"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "argos-ci/argos-javascript", "label": 0, "pair": "d9d8cfa38981847f", "unit": "864f1800617f5218a2300ce390ba7922fe1bd0a182de115952342b61d627bb16"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "argos-ci/argos-javascript", "label": 1, "pair": "d9d8cfa38981847f", "unit": "fdcda2307e20ef1b9247b7082a85337f457dd1f4e648bb8264077bf983c03271"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "arikusi/deepseek-mcp-server", "label": 0, "pair": "1f73ad2576d52a64", "unit": "0bc9634940734cf7f41628f220032fb7e2fd2ff5b584afc519875e90ea0be5a4"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "arikusi/deepseek-mcp-server", "label": 1, "pair": "1f73ad2576d52a64", "unit": "ddb59b2dc39715281a6d48b32e7ca4ec0354b85b953fff55139579f83905b7d6"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "arthurkatcher/google-maps-mcp", "label": 1, "pair": "a7326e6daa507b79", "unit": "793ebf4c8e0f46d8ce3b3d090d2f6290d4167931a3b574f8269b90008e3dcde7"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "arthurkatcher/google-maps-mcp", "label": 0, "pair": "a7326e6daa507b79", "unit": "b8d25f337fad527544c33c82f473015e08399457276fc480477f3cfcfc9c5b8a"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "asymmetric-effort/nogginlessdom", "label": 0, "pair": "211ba072bdfa67b7", "unit": "3217f9d672a4a1d78005c8fb66e1bc60a542cb26bd8c46903402c765c123d60a"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "asymmetric-effort/nogginlessdom", "label": 1, "pair": "211ba072bdfa67b7", "unit": "8370f9624baa0d144e56b8087eea897b98e5b2afcd6eb3a029d03e5b4b69d584"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "authlib/authlib", "label": 1, "pair": "baccf350d5cc9911", "unit": "50a3ca70700c8b41874c43ba94e6b7ffe0aa28b8930ff941a40767e1057065b5"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "authlib/authlib", "label": 0, "pair": "baccf350d5cc9911", "unit": "62927a118d3662c81450e19d49f4146b66f29a58d55ec0d1606ed521b1883f12"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "axios/axios", "label": 1, "pair": "697f34e19bda57f3", "unit": "775b6d900ec1ba71e4082ed49bb94439f38066628eed91461eecf0658caee78a"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "axios/axios", "label": 0, "pair": "697f34e19bda57f3", "unit": "9e855586c685b3c62ffe32407a82966661d5839e2cff742f2676acaba645a97a"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "axllent/mailpit", "label": 1, "pair": "143b900b99a46cf4", "unit": "200f96cd784ff5c39d8bba22883f22bc8a7c87c6970a7c871802f48a45548337"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "axllent/mailpit", "label": 0, "pair": "143b900b99a46cf4", "unit": "7a0d62ec424849ec4681f4333fbb077e8239054d1e964bd99b40847ad7ccdb01"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "basecamp/trix", "label": 1, "pair": "4b4e81797ca48cbf", "unit": "a4d17d85973d024a615b89334ad5a2f46afea660ef5d0bcf2b2196c998b1caf2"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "basecamp/trix", "label": 0, "pair": "4b4e81797ca48cbf", "unit": "d604e460d8f5291124ba6fab5f9c94c7dfda61ea5f6808e512162334d6d6db4f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "beaugunderson/ip-address", "label": 1, "pair": "bcbd06e53147628b", "unit": "0b5cfe2d64771e30e50ff92cc409b8dd5e30079c252918ba03774cb5198a73b5"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "beaugunderson/ip-address", "label": 0, "pair": "bcbd06e53147628b", "unit": "b805976c35330452af6c9f814ff4617b7b9ad0e82e1fa4440a443f93f0772c4a"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "browserify/static-eval", "label": 1, "pair": "c12dbc8a50f7fe9b", "unit": "04beb8a0fd054b92c2dd0ed83907330fdee6124d86114df948ac72a297a7d51f"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "browserify/static-eval", "label": 0, "pair": "c12dbc8a50f7fe9b", "unit": "c5029016a6fd2e3f9df59dba0ce3c173bbbb0d2b1de22d151086e01e1b272119"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "0e213aeabacf4205db024ea3c557e720c3aba911424555784917e967d6404867"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 0, "pair": "e8bc65415bd42e43", "unit": "4920230ef773595a9fd9a92a090dff42b48499a1dc5fc8831f941dae55cb2538"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 1, "pair": "e8bc65415bd42e43", "unit": "49209a810f207eb61382e2ceeab7076936838a5652546f21c187d066e220160b"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "9e2964884cdf281fa3f18ee3481aa62f9a8a0a195d605da2df5bd71876fe311a"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 1, "pair": "", "unit": "f9cc6f1bbcd59f5dc0de8f96248f722ea0679f8afa47c53f70832151ce7616d3"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 0, "pair": "e69463f945f9a0f4", "unit": "5feace7b28e260175f80c682e9009025aea6fae3dc75409cabbb955b560d0372"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 1, "pair": "e69463f945f9a0f4", "unit": "799a4f5a5b9324e86c90752f0239f059984f44e8ff1ec0dd518d0d9413e34c66"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "cakephp/cakephp", "label": 0, "pair": "71f56810b8ee6947", "unit": "c659c0e717b7dc1084e1178288f6c9b2e87fbf0ef518ba3bbaedb0fbadae15c1"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "cakephp/cakephp", "label": 1, "pair": "71f56810b8ee6947", "unit": "c8d2e035a6099fb775c8b4ecf00196c1488dbddd9eed3d2d268a4f1d2567b194"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "cesargb/laravel-magiclink", "label": 0, "pair": "768fc9db8a75a351", "unit": "adc6527469618f72b1dd7776814717cbd0056a0c625e05bef74199171ce73f1a"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "cesargb/laravel-magiclink", "label": 1, "pair": "768fc9db8a75a351", "unit": "da47663a57e370a75cc65b810b8a380aa26d12dcabb1d9b3c02641398ff7d348"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 0, "pair": "fab4ce020049724b", "unit": "b9be90242be627aa46462143ac85940a906232ebc0b47776b5729475b27d215e"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 1, "pair": "fab4ce020049724b", "unit": "ece12ea7d87b4a12c4da30faed9eb530b51fc879c271b68ba5304b230cda8e79"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 1, "pair": "9126ba83905eb8b4", "unit": "d8dc50b7bb40bbcfc3f3d4f1f8d3a4d17eb1c662d5f759efd18656a482dd2e32"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 0, "pair": "9126ba83905eb8b4", "unit": "f426f9307621b3f2d6a8d1669cdb4c19e962b94e34fbb394d4660cbde5921c88"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "commenthol/serialize-to-js", "label": 0, "pair": "7ac5e21d9d8f4c47", "unit": "1d1efa2fd7323c264290d4d314c957c853e1c7a9c877a54d6f03ab7d2af2ca86"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "commenthol/serialize-to-js", "label": 1, "pair": "7ac5e21d9d8f4c47", "unit": "b5dc78c7ac0b76803dc98904142c1139083fb9daafb5bff40e7b5c3f14ee03f5"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "0894a7212280bab755cd9d93b1f0ede2c51e4fca76b76e5ed9e24ef8f8cf0b9c"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "02694b5fb85e9e52bf5eaaab4e0b628f1f44ff3a8d5ad026cc10afdbc4222eaf"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "db9a3e888780501dd3c2c125af2f9d47f68ebf02f0e0db8370b2ea09c6f9d070"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9af96b099dedcc4005e5fcd19ad78f73864e1cfbafbe6953b4edb6260957943d"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9e7c2a786068a936a6c664cc229d6f39cea1fe57caf0806306f0cfc44515460c"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "cps-it/mailqueue", "label": 1, "pair": "07d1dc4d299a7700", "unit": "0e664e274dfb438d2f721edc81f8469ac48387dbd5c1b43973cd7e651db46db3"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "cps-it/mailqueue", "label": 0, "pair": "07d1dc4d299a7700", "unit": "da55f60378c67e91683b9720f2fa0e6659e28c7b392c050278b40713e47c84a3"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "cstigler/node-xhtml-purifier", "label": 1, "pair": "749eee05a507fe93", "unit": "24fbb1dffcca083382a23e48db232e0d83bbbf2ff2b325f0dc0f4c72dd1a16bc"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "cstigler/node-xhtml-purifier", "label": 0, "pair": "749eee05a507fe93", "unit": "640e1626f8d7d9d5c2478ede48cc7b2780370bda98ae114bae102bba892da90c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "czlonkowski/n8n-mcp", "label": 0, "pair": "e36bbaf8f3cfe65c", "unit": "14f7c65fb5621ac5e2068c8695ddbf482103e4320ec3b36886f6e1f08cbc312d"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "czlonkowski/n8n-mcp", "label": 1, "pair": "e36bbaf8f3cfe65c", "unit": "c96dd379dec2ecfd99051d7a20e6ef5601821f93c46eb34ed087daf5fbed2d47"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "danger/danger-js", "label": 0, "pair": "b81c8c63cae892f6", "unit": "5982fe7fa1ea6bfc1c8dcb8c71cd4410de475d25cafa324147649f26a1ae1324"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "danger/danger-js", "label": 1, "pair": "b81c8c63cae892f6", "unit": "59916deeb8ba14922eebac61c9eac122e15d1f7590ee3cb1b97f019585fb798f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "danielgatis/rembg", "label": 1, "pair": "bab3a167ab5ca505", "unit": "5ebea23aee96c80fa0d2d9239964d99614613c4ae36f709811bc19e61ebc720c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "danielgatis/rembg", "label": 0, "pair": "bab3a167ab5ca505", "unit": "9bfea8d7a6ffa6d79d16f77fc355fc1bb605189513ebad561ee523d577a1498b"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "datapizza-labs/datapizza-ai", "label": 1, "pair": "5a91049be21aa611", "unit": "1ab9936627aa59c3a60ea44074ad44b68aea4f9c04f14c1949dd9c77ca13ff0a"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "datapizza-labs/datapizza-ai", "label": 0, "pair": "5a91049be21aa611", "unit": "8df72b9eab3f2fd858c40312d0a806cb67c620bd62873826c8732ba8795d9cfa"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 0, "pair": "", "unit": "115d87f37bf1d4ab91fa3010303e81b127f84b817268b60b8fd4b9934efeae4f"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 0, "pair": "2b8101c89c839614", "unit": "7bb7d858a670fc51f81e08c141c2bca3b8b7524254ab679e33b22ce79536e895"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 0, "pair": "c23acf4f38945093", "unit": "82bdc0b6bb0c19e342efa63af0c9014a607461d0682cdd14bd5fea33516fbd50"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 1, "pair": "2b8101c89c839614", "unit": "886285661b88f14e00ff8e863b31cf7b7afb303ad4aa4aa13b1cffbf1272d5e0"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 1, "pair": "c23acf4f38945093", "unit": "bf74aae5a48156f183426a80cafca6da219ab13b2ae4b40fa88fca75acb72980"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "delmaredigital/payload-puck", "label": 0, "pair": "d7cc37a3a54315cb", "unit": "4aecd43c6e5b81daf7aadd6a6e22f22c17c08b652a2f5121a49320ca1057b186"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "delmaredigital/payload-puck", "label": 1, "pair": "d7cc37a3a54315cb", "unit": "88123ae860be2fdc28fd6a289cf26c3e0af0689f455cf8c15098e5dc76e5c8a8"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "dep0we/atomic-agents-stack", "label": 1, "pair": "5011004f06f1e217", "unit": "6c48682d1b02ff2559d92346d192cc9dbe6a0bbe2ab2c146653110420105ff1d"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "dep0we/atomic-agents-stack", "label": 0, "pair": "5011004f06f1e217", "unit": "d3592260108a838246e0cfd5ab6d48bceb0ef870e223565745dcfd68f1826840"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "devbridge/jquery-autocomplete", "label": 0, "pair": "81caad96b221c2b8", "unit": "3c2dc720ed159e55629ba737665d93859def9790427289567a513c83c133e236"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "devbridge/jquery-autocomplete", "label": 1, "pair": "81caad96b221c2b8", "unit": "e8b9595a320776c8e663fd0b168193c1e64e36a9052784d1219f2c4a91d647f2"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "dgtlmoon/changedetection.io", "label": 0, "pair": "f9b6599018ef3b60", "unit": "7558d6ee39a311c5459e35fe6f7b12917b7f843b21ab1fbb5e485a07f53b7d78"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "dgtlmoon/changedetection.io", "label": 1, "pair": "f9b6599018ef3b60", "unit": "b1a6cf450a074788ffca2bebd880435abac2c376e89cb328d7a22d277fdccad8"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "dicebear/dicebear", "label": 1, "pair": "eaf2f03db2c502b5", "unit": "38dbfbf745d7cbcc52c6fff83248d84663273fe6a1f1b014eac7776c4bb8f074"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "dicebear/dicebear", "label": 0, "pair": "eaf2f03db2c502b5", "unit": "a84d848ef56b72fc15872d88acaae1743c9cdf88095d13b1869abbec6a0d8a8f"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "docling-project/docling-core", "label": 0, "pair": "7df48bca2c9fabfc", "unit": "7b76ce8f55d61071adb367018cc3e50f04a05d08c7c4919c3806fc1cc8bf41a7"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "docling-project/docling-core", "label": 1, "pair": "7df48bca2c9fabfc", "unit": "c82ea0adc9c8366aa2b77ce73772038676fc37ff50879e873ff397958fdd7ec1"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "7f02a09b8d8f2321f2cdade74c2fa1d52442e006149a749a8be768daaa62195d"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "9b5e439943ce5c2b04d6201b7a843634aa1b13b726813c23450ff54ce03f3dd6"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "5b836889b4da4921", "unit": "a84c2ce4a4eadf265d4481ad7f6864edf1d7e3084c6ee32cbbfe454bb2946bfe"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 0, "pair": "5b836889b4da4921", "unit": "caeed7f097a3311fe2e84efd372fe4b4c140f63a80a9f07ba0b61966dbab15e0"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "electerm/electerm", "label": 0, "pair": "ac526f457067fad8", "unit": "50d8bafd5189fab0429bd4135d9ed400cc2fc14706226a2de98d65746ebfdcc7"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "electerm/electerm", "label": 1, "pair": "ac526f457067fad8", "unit": "9532b70b0df76a34170b57c74deafaba54d5d2f16a729c39b0dfd3eb2022d899"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "elyin/gemini-bridge", "label": 1, "pair": "c97a971a18f85244", "unit": "95c80c503ed43fe9a0370f89ea935d47badee6a9625548b9dca78e04275003dd"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "elyin/gemini-bridge", "label": 0, "pair": "c97a971a18f85244", "unit": "e9810872beb9c158f54048392142c2f701314f854d6dfc748553ca36ed6c08d0"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "c2a2c2d85114a97f892fb1e5500fbd98622457a8c46451e1f9e958ab08a40318"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "29d8be8206c88ba7efd86b56a7dcf8b655e33b4a792c2d18b028c92c6252daab"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "esphome/device-builder", "label": 0, "pair": "6a4e0bd705331747", "unit": "6db80439d89b54f60f836c415dd403fd09d21cf630da81435d4d2f9cef5d883c"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "esphome/device-builder", "label": 1, "pair": "6a4e0bd705331747", "unit": "bf257e4f0812012be330fe119d07d29253d291161f435262787f491762c6bfb1"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 0, "pair": "f96f0fb245948915", "unit": "0954e6931373915c204b9ec8d2b8ae6297a95667bfd41aadf9035625ad93967f"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 1, "pair": "f96f0fb245948915", "unit": "5e09cdae2504e834d591b6f92a808427f07550364b5419899f7e6f76084fc377"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "ether/etherpad", "label": 0, "pair": "bf11f138fe39675e", "unit": "56f0d0b156992af56141d6368fdb0c5376fc7a44a03991d754f97202b9d77008"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "ether/etherpad", "label": 1, "pair": "bf11f138fe39675e", "unit": "8271af5017894167556893da2070188060d06eebc87f27db45c46234c359631d"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "fastify/fastify-static", "label": 0, "pair": "9d937c2dffb4aadc", "unit": "01ac12bb06e660d0aeef8b1e26dddc5c99d52cc333f7af6d1981eee2818e64fd"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "fastify/fastify-static", "label": 1, "pair": "9d937c2dffb4aadc", "unit": "9612edf0f927048335e83e9925f01be9aea9b772bdbdc568ad9cc0a17055662a"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 1, "pair": "d4f648048e45f2ea", "unit": "08807c8110b8c1b8635a6883c229fb91f2b9566db8b6c252448a0db7b2ddd47e"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 0, "pair": "d4f648048e45f2ea", "unit": "10869c931053bf77ecaa98a47f56dea27d643d1abed2a4758bcf48a78547ccc9"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "662e1f36c230d7f1", "unit": "0b73332b25413902c26d5b5dee6767465eed99ff71a814582988ffe0d8ec1a30"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "8c8e46d0c7d361b784baff9d70f38744b2805ef1ee0aaacaa616bc20a2c6f32d"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "aca938682c196fd4c85b39b89eeedb97c35f6a5a017651fe9c03c5b727cab7ea"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "cec00d1509a129364da6ab8537fb3f433fd847e26d073c90311ccdaf7a1b39e7"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 1, "pair": "662e1f36c230d7f1", "unit": "e0f58a7ac5627de3e7b6cfe5add41d18b0f64b998c1cc788d42eb2cdbbc3fe41"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 0, "pair": "2c251a645094f389", "unit": "0d41cc634771048f72d07c4aa9feaed669d0bfe3844d61dcdc525a0f899617e2"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 1, "pair": "2c251a645094f389", "unit": "ca8288d0b8b94dc6b7e4c681936ea610d295fe949cba5f64f09b6de134ef9875"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "forgekeep/nebula-mesh", "label": 1, "pair": "2ac5ec5011615c9b", "unit": "2f427f403af3cb58d776437736c2389805543413fdb91aad98979158c6d00de3"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "forgekeep/nebula-mesh", "label": 0, "pair": "2ac5ec5011615c9b", "unit": "cddc8c5bed0841b80e7b20d0b378e858f8e4ac37fd7b890ce76611d989857f92"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "frangoteam/fuxa", "label": 1, "pair": "", "unit": "0c7830c2912ff285855d8a3c8e9b47db4ca4af840baa66d4b5d3f4422f46f95d"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "5383a8c2a78fabd3d17a5b3021125953b5ebbee61f8900c30d016c56b1a3bad6"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "360d8bfd2f116e98", "unit": "7c1989a4ec3622bdaebcf1161de850f22cfed041cb38830ca1ded4fa99d40466"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "ec8c706cc2b20f36", "unit": "848688dcd0fe8d46e6762cc8aee1d82e8fad8a2a044c94b547aea11049c85bfc"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "ec8c706cc2b20f36", "unit": "983cc9231f91baee4e2d626193f3b88aa5b09907f6faab870949bdc9c336b816"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "e4ad86416d5c6668", "unit": "98ff1dff2afe80709b8e61a9c95cfeea12b2110c13b7120478268b4762e2c992"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "360d8bfd2f116e98", "unit": "a1a825d23e34757c8e961e43ae791253e62682e130acdb846b58118588006a98"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "a7fa4207508b3989baabc7a1f0ff2ea5229b366c1cb106e7fc945134b8b106c8"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "e4ad86416d5c6668", "unit": "ef5c53f3114d0057e694df13dc07cbcb532c3b0d59150eda269b87b021d7a6e8"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "getgrav/grav", "label": 1, "pair": "9ae575048504c690", "unit": "176acf077fd58c3e6f940753179ffa3e8316661f355d94acb5e0345b51925ca4"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "getgrav/grav", "label": 0, "pair": "9ae575048504c690", "unit": "228b34ebc2090b5239c7821d28366ddc663e7118d015a79b45dca6618d5490fb"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "getgrav/grav-plugin-api", "label": 0, "pair": "d6edb5496b3fdba2", "unit": "9acb8017ebadc480515a664181c427f37359edcc1dc6f344f368a87e0c191b3f"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "getgrav/grav-plugin-api", "label": 1, "pair": "d6edb5496b3fdba2", "unit": "c1bfe63148bc700c812fde95ad9f83d456e4fa04e10c6f5394072a08f73174c6"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "gitpython-developers/gitpython", "label": 0, "pair": "27e1c8e8ee93a892", "unit": "47fb89b132d6de48674c37965579e727d922bef1796db70f8b9f862ac47cb9a5"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "gitpython-developers/gitpython", "label": 1, "pair": "27e1c8e8ee93a892", "unit": "79887f0ca8b05d6599d55167d8a9faf67bdb2254d17c143141da44971077002d"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "gofiber/template", "label": 0, "pair": "a3fdf9ce3441ec86", "unit": "653236018929ab8534c27704a20ec5d358047bc66e451620d2ac32081c91d333"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "gofiber/template", "label": 1, "pair": "a3fdf9ce3441ec86", "unit": "b70ca2c43f5929e7b885bd728349e721fd85d7605e2f722a7e8cfe0f0d477b0c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "groundhoggwp/groundhogg", "label": 1, "pair": "", "unit": "92400e83890882a66ffdea1a405e2354a195442e176d85fb707b0442c44d45ca"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "gtsteffaniak/filebrowser", "label": 0, "pair": "f9dcf3a0fb0f0a14", "unit": "7ca6dd4f673976b000a657f885af3463740419ce41e14f972af0c5483cb75a7d"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "gtsteffaniak/filebrowser", "label": 1, "pair": "f9dcf3a0fb0f0a14", "unit": "86c1db3a5e9f843d6546146449171e0378ac61d6c8f44c6dd72fd379cdbb188a"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "h2oai/h2o-3", "label": 1, "pair": "3e71f79348a6b770", "unit": "200250ebceadde4cbdfec2802a4f9fbfa220dbd7fa4c2dd5ed717fb1f3ea7447"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "h2oai/h2o-3", "label": 0, "pair": "3e71f79348a6b770", "unit": "b9faa5007c8cbb67ada3f1f026ca59e0ace6fbb9c3d1f1369d4521a7e323e7da"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "h3js/h3", "label": 0, "pair": "8972a0f56ee57477", "unit": "6b987df2960828dcb1b2b3f190a903a40361d58117d63321d671aecc6c2cb570"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "h3js/h3", "label": 1, "pair": "8972a0f56ee57477", "unit": "ca3fa87dd37435c01a7121119912c1a1e261259c11805c0b66e7d4b315640945"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "handsontable/formula-parser", "label": 0, "pair": "6144baeb491f4a56", "unit": "f5c22ce104e9593e5272b6e8b3c6f3b61fa4578552ccea80e6021ee90bdb3ffc"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "handsontable/formula-parser", "label": 1, "pair": "6144baeb491f4a56", "unit": "f687e7e95dbe47f602a1e0002a44f94fd48e5b1c33f77ecebc6c2d814fcbf450"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "heartcombo/devise", "label": 0, "pair": "2ef4471c66e5e4ab", "unit": "ad7c8e368fe1699a044e058d5c10b41bd9fc0f91c3f3c384c5dac40af39e71f6"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "heartcombo/devise", "label": 1, "pair": "2ef4471c66e5e4ab", "unit": "d888baf7194bbe7158cb3932f58b47a9a235351d388d10df82a130ddd54c2d46"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "honojs/hono", "label": 1, "pair": "5628c54152216646", "unit": "47d081c9f6060218967902f694e61c88f64f57c65bec6fc3073afeb0b77d117c"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "honojs/hono", "label": 0, "pair": "5628c54152216646", "unit": "e1297bccf9c3606e0b41d762a69ff5d0a33ae4f7f1506071af6b9cb4a177ff56"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "hs-web/hsweb-framework", "label": 1, "pair": "e52aa814c36ace1f", "unit": "2d6d74364f1982e91f27f1c995b8699df1fc8f8dce6e1e2cf274bf01b6e9494e"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "hs-web/hsweb-framework", "label": 0, "pair": "e52aa814c36ace1f", "unit": "4727471fea2cf0a92d43b4b1503f1339a7dde87f872ba8064dacbf855c57705b"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "huggingface/datasets", "label": 1, "pair": "a07929b3fc1cfc28", "unit": "0637029194bd7378237b4dc944dc5cf9f2b63db9796f27410a74a9d567b66dc4"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "huggingface/datasets", "label": 0, "pair": "a07929b3fc1cfc28", "unit": "ffd286c8588a77929639a89e4a742e15306b642af14aa906c721165df804ccd3"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 0, "pair": "6548e1527e82bd99", "unit": "5542613f6d2eb9421b8213b8bb864894f0ecf9fe64945681019230ef0d75cfaa"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 0, "pair": "", "unit": "6c2adee34e9e8384535ec1073eda60e2ab528ac77c36747802c303e1f5e9ffe8"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "7ce70e9b20107cc7ebb87d173da765fc53ffd268e521749a06a9a1f38d746845"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 1, "pair": "6548e1527e82bd99", "unit": "c3e54975464323e58ec96ee4d83e7bc08baf94553d553a8dc3fb28cd9c70ff21"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "d3173c1e43eb6402a71ea2dec1592913dc078a2194ba1c1d509ab129a3180f6f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "internlm/lmdeploy", "label": 1, "pair": "0924befc49c7bb18", "unit": "e7e4872db40f0420547ed622fbdd7a072b64a7ba466d1ccfff68e691cee6ad99"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "internlm/lmdeploy", "label": 0, "pair": "0924befc49c7bb18", "unit": "f7d95f43fb2df20ebc4c37fceabdd00ffd8159f404ca384481f1764252382388"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "iximiuz/node-diskusage-ng", "label": 0, "pair": "d4ee6cfeba71e398", "unit": "e79582de3031bada2617a0667ef7ff5cd7183d4aa885f55a665cf0ec88d17624"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "iximiuz/node-diskusage-ng", "label": 1, "pair": "d4ee6cfeba71e398", "unit": "fef35272853e7f3692efd54daa01c12862953352fa6ccbbb5e3f7fea70c63d78"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "jarofghosts/glance", "label": 1, "pair": "772e18821701d60f", "unit": "89a0496c7663bbfd67d660c9f481c12420da2dd1bdb29dc079735ee7a0a49626"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "jarofghosts/glance", "label": 0, "pair": "772e18821701d60f", "unit": "d3241ceaa50a7d23a4650e6a2a55372b08ec96b35b0bede844bc40e16e818ebc"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "jas-/node-libnmap", "label": 1, "pair": "250de1302c7e541d", "unit": "5d03535f2df2acdbc319d9e6b20da65d51f3e810b80f3c8f89b5074a971a3f1c"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "jas-/node-libnmap", "label": 0, "pair": "250de1302c7e541d", "unit": "78f977cfc096e34b4eada7f798ad2973ef45728d0d01bfab6ac9e2a3257827ef"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "jashkenas/underscore", "label": 1, "pair": "c10bfb006f2da55a", "unit": "1ea4b86dbd84581b1d7b7cabbab9f28c8b213416a60778879666ff42d23fd7f1"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "jashkenas/underscore", "label": 0, "pair": "c10bfb006f2da55a", "unit": "7647021f742da24c660b2d407886771786e957076c4cf336def560a570dd8778"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "jellyfin/jellyfin", "label": 1, "pair": "a73421efb106ab0a", "unit": "1f6f207a758582de0c0e6b331b19fe119abe18727d45ccbbe09aabb795f3a292"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "jellyfin/jellyfin", "label": 0, "pair": "a73421efb106ab0a", "unit": "32fbdad39eaf5177db4ad0a11d53e785b30bc7b2376ad9d460f29a08429bd0c7"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/cas-plugin", "label": 0, "pair": "0458e3fd444409f9", "unit": "b521216bb4106738b3571ad3902d08f6230aabd88cf3a8f9f26f1c8ec01304cb"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/cas-plugin", "label": 1, "pair": "0458e3fd444409f9", "unit": "c87d2871907798fa2e73d31381e9d63089b0388033f096d247373fec07c2cb36"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/google-login-plugin", "label": 0, "pair": "6bdccabcf5b9fdc9", "unit": "3d6ddca5eb264d88411b86755909f1d2abddd09c129cabd54c96ff26b4105316"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/google-login-plugin", "label": 1, "pair": "6bdccabcf5b9fdc9", "unit": "e055b898802279bc63204cc601e29f7f1018efad7228cab0cce00ea5866ecc7b"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "jenkinsci/mcp-server-plugin", "label": 1, "pair": "91bc832e70680c92", "unit": "1060c63ff0fae3af567448f3d4c60d9c159b61ccc358964243eb152a9ac38865"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "jenkinsci/mcp-server-plugin", "label": 0, "pair": "91bc832e70680c92", "unit": "369dffd1e730d7997813adfbebf1481cc68987cae0d0085e47b91a26d5330cf8"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "jihong88/suneditor", "label": 0, "pair": "137fb3496a3a75f4", "unit": "7e8b6c484999229b23d50d9f5a4c4c5b2a8d954a5d64230981b1c6705b760fbf"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "jihong88/suneditor", "label": 1, "pair": "137fb3496a3a75f4", "unit": "c461154e4dbd74e35a4ed2e85de7b076f0582beeec459be28104c38f616c7565"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 1, "pair": "a4144258dffb3b8d", "unit": "7ab301e1614979f44c353e665540582493f78ef4bfc41209562c19b900710f8c"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 0, "pair": "a4144258dffb3b8d", "unit": "ea95e8387dff32ded9dc7b04c11a76a545b4e2ea6db0076b9b13971a59904bb3"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "jmespath/jmespath.rb", "label": 1, "pair": "858330e01d3cf9bb", "unit": "033cda2b7cffa5a5fa6abf0b7f2cd27ba78d916eb3cab8c79decccd8776f4a3f"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "jmespath/jmespath.rb", "label": 0, "pair": "858330e01d3cf9bb", "unit": "d44c4555a5ae19aa17b468ec5cff56b33a0b221df52e04d8abcb656399499dee"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 1, "pair": "c6a49339d4f0c798", "unit": "dc2e4cd5bd963643702d2b832ae170dd25744162e4076e36687256a09e69d089"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 0, "pair": "c6a49339d4f0c798", "unit": "f83cd137b85e651ca5299c5a42176b797ed4964cbef31418c96d63c52fdb65f5"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "jowilf/starlette-admin", "label": 1, "pair": "4b6526d132e9ac86", "unit": "8ec3596444f3045684242840e9a1df19879383f2733aca8f8af6f6cf8a96d388"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "jowilf/starlette-admin", "label": 0, "pair": "4b6526d132e9ac86", "unit": "f3d39162fa14446b207aabe7bf3c913dc32a9637ac83352ea5cb2fdbb43720a8"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "jsonata-js/jsonata", "label": 0, "pair": "80abf1d5c4a3f6a5", "unit": "3df3c6d9941352dd2861427da56b94ee335f308d0a2a27c6f745aef60349c80f"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "jsonata-js/jsonata", "label": 1, "pair": "80abf1d5c4a3f6a5", "unit": "4fde61a570906e658d9b14db0e5d1f5acc0c7489951421bff2afde25033ff73c"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "jsonbored/gittensory", "label": 1, "pair": "5ec37c0f475001ca", "unit": "5d8175fb21a4649bdc8730358e9b87adb90b2aafc4da5fee203829663e4f46a0"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "jsonbored/gittensory", "label": 0, "pair": "5ec37c0f475001ca", "unit": "af845e9e332555096dd350401977b2f07015433f3db8a040891d9470f3f9e355"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "keras-team/keras", "label": 1, "pair": "68d44ef338ef3470", "unit": "3020ac6c3186d7974953b1c4da65b80141b81aca5203436ae7a2d7aca0870944"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "keras-team/keras", "label": 0, "pair": "68d44ef338ef3470", "unit": "8fdb85a6bfec89f0715ea81e83fd91616f92c59954c092e79272154f202b7488"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "kevin1024/vcrpy", "label": 1, "pair": "c74525e844650de2", "unit": "2e444ad34ee5f47d73f0cfad0a10c70363337b4ddf9067ecd1b8072d8232652c"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "kevin1024/vcrpy", "label": 0, "pair": "c74525e844650de2", "unit": "c492d325f5ff82b5f817aa4be1440cec8595054dd0719e07a58ac5501b2ae7eb"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "koajs/koa", "label": 1, "pair": "111a5a80a3abc2c9", "unit": "c7ae403742450deb2bd051ad1231121d2117ba56fa72a038c82d58d207cb3870"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "koajs/koa", "label": 0, "pair": "111a5a80a3abc2c9", "unit": "debe34831afca6f8220b9952036a2d4570babdb5e321a2f8dbaec661a016bd50"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "9950b85412b021ebabb11eea4c490a660ad380d14c673a87c3fb568ba4bcfac9"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "656df5ff8e69dcdb1f9ba1ed52d734966eb29773bc745587483a355743b8eb5a"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "91652a4e776fbce70cde9e4a99c6da5e17da7f8d6d0404867c61eec04161b6cf"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e53bb3a217dc4737e8dd03671de4f6a895599024070a985a268c65a034295e7c"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "3d7aba22444811a337abcad0e4fc62990bb704d3a10e72d893342fa94699026b"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "5f18a3026a51f7140b7aaa3aa8563e8b6fdda6065d95e19e25b1dac4b9ce0e00"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "0fccc306ce80cb4cbfdcde0a2e56d637bcefa9dd30423d6c54a4f77860afe740"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "1e2eaa45661345f1b6759d968d7696dd0d96d6fba1350a00e4d0a2ebaaa2e391"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "8b9a8ff0e32a327df75eb8a1811464d5f34690079d5930139dd917063bdd2809"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "245d0e939b7c1ccf098c798492009e3a73ea17400fa613dd865cd369d2e7f383"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "653c48df493f31c2d01c973d1e379a180fbe6a6edc83a500bc62509112175f41"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "3719fe21d45b50b35458bcd2e263e4baee67adbd3c98adf0eaab6bc6183c5651"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "1209a6cc8c1548ed83d4a9a09d6a27f706a23e69cf0c261fc5e3a27536a83306"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "362c90495dc468bb732c71f62d71ec308854161de31ad18d2a870056d947ddb6"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "b52e53fec9707b2b442c4dc7a41d376cf76946d2c45c2eb600c9477fc6baebaa"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "477e3b90b92e622888dbe0e9d3a23b31271ddfa3104a5ab326eb9e9e765aa01d"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "660c63c4a73bfe64088ae3249e40d9c5d69a6f8a505ff671217c5708a345c5b0"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "4d60c386c801bedb61c1461fced29a3cfb50b09d6435e8617c60398343719d93"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "f1fddcc1354a29bef55610ee2b73eaa440192f2f18a0d22d08057b9d1e1b61b8"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "6d9886c38221b2f9aacf83749218ac5ebc354a6fcc4170e6c78bedb92b88296a"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "5550fec4a81c4c9cb22d7312d8a63e1fb2b03c4aed0c1f316d44c1eb27511bac"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8ff87dd04140f0137ea1891f220b1276156ce10a58be5a49d0c8e07be531ffb2"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "f24b19cc0f67247eccb274521a62741fef8f4729f0ddc9b5c1fa561a4947182a"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "83523c9d83380d4cdae593a593d361493e00d18e61740f84e762aad41b991e18"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "c6c9bde6630678377d250701073c5c223c7892e6ff8f2f8e0293dea22c5b5e7e"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "b9b15958973c83f8d7b58372ab5ba58eb191c400125579d5ae8c5b52d5d1368e"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "b95ec12e4e126ad558e944e2c260116ed6a63f5f3ab97fb5cbcdca75a931ad73"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "38a5545ac9ef67e15db720d1396a2d5915f9d06d6f4772012e5bc95f33770376"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "c8bbe9be91c976be97b7ca387048a7d4537d9f6fcc6cbad0a541d1080629d5a6"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "ffacd53f1bd75155e63b318fb9cba27ec03e754841e1eb39d1d4fa0b2c3dfca2"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8fd920175fc43b91b4397acffacb10ab8f7fd0ee38bcf2d2f3ed06a3fad0dfe9"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "86c9598ebe3dcda261a69ae545513f6ce73101f3980066e4506ece99bebea84f"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e5190fcc57505116f79a31c51aed69f8a1eea13a8ae3e7265f44c72418277ef9"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "2b807ff54e6c47e941124c16d0c27c691cef54f2203060a3a6ce812d478c5261"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "401aefb4b4791301ace2d770d321c6b26743b0aab7b96f8b410254c704be6d2e"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "22d186c93498e21418b6c1d0547a73374b13041d8afc89a80d9540fa658b6b42"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "dbfb6df14fe37f8c0efda5f16643297fb0550dfc5ad5805d4e4b844d967773b2"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "f0da8cfe4ac1e586aa7903d72e67b44500bbd5306d72b4b342d10613efc69222"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "2b8fd73943f456da266434470333a7775f55d9a41953e97a000fdaa3257b6a30"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "a612091f452e60f28f44eefec641175170dc5db04456a39a099cbb13c50b4d2c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "4c6600facd83bbf234a32d9e28ac494fb516df6e3d9e9dc28d720ff35252d067"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "fb0ea8cab8c9430d22d1b0fdd65a69146fa95fbff94e720b4861b96538d992c6"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "e81f91a56058ac1473579767abea57064c53d7d007797a854314814e3c35ee74"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "5851ecd96a449f98b05f243bfda0a7cf3af8612b27f53c414ae1f4ea67242a7f"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "328621ef2a94632e5d37ea6267d17b766233384cde35c07e29a4eb672d40ba28"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "04b2eae2128608dad460ba8275142a41a44e32c4243a29e2a6fb2c9ebf3d8f1a"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "33337002ff5f5a189610b042522ceea254c9d107c1f24771b4f616a0f96f636e"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "02c09a783cbf058856fc2b448406f3ffa6fbfbacf9b70f4e5e22fd741a5c7699"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "956260164cb21a47c499e5598491031ef303418943f5096f225a87162b588574"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "8bd8c5aed69eae3019cc2d3654b36bdf2b31435e6a020a389cd103696a2f874c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "1d8397ef4a9fde7b48e2e1e871bba0f960a802cd7823ecfc10e9dfaddb955ccc"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "c9db92476032eb4296b950a1eff8fa4aaa26f11542a01a5cf10c7b2e32dc6433"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "d3e0d0f9b0f4b723080989b5486395cda0a5b186a674022ea70e7fa7ab561323"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "44ff3b3e61c02744830c4abea872821ca7e0f94950c082740c5c48ef13604e94"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "4103e979a99233c3e71fdfe9f1cdcbab34a2c60b5351b75c5ff5dfc287b962d0"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "koxudaxi/datamodel-code-generator", "label": 0, "pair": "f73ecbfda2bcadef", "unit": "53e596dabf90b2992d5b3e50394791859b393beedc8f84e00484e37498627ff0"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "koxudaxi/datamodel-code-generator", "label": 1, "pair": "f73ecbfda2bcadef", "unit": "6daaccf64706e876dd505c65a62aeacc1357f3470a5cef7b1ac874032a6fcf99"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "kozea/weasyprint", "label": 1, "pair": "6768bfae38c6859d", "unit": "f376519889a00950f5af67eb15c639306ea70da7235d3be4a94dbc19cb096cf5"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "kozea/weasyprint", "label": 0, "pair": "6768bfae38c6859d", "unit": "f3fea4e334f49ef6fe6b17121fef553eb8e33a9ffa13e6d6cbc4c448da996a7c"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "kubev2v/migration-planner", "label": 1, "pair": "d670253fa209671c", "unit": "1aeebe2e440bdd70525c31f93180f33be93f820db5ba46a2e59c371ef83ee794"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "kubev2v/migration-planner", "label": 0, "pair": "d670253fa209671c", "unit": "b11c8a8a24d3809c0507228568fe7cc05ca0a22ed28cb19019adb13cb619d63f"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 0, "pair": "33260f3e6c8ea6d0", "unit": "7ca409c7dbc1c38ff005624e212efc6d424cb421e27b482dc8dc0d6881f4c11a"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 1, "pair": "33260f3e6c8ea6d0", "unit": "fc364d7ed77c7c0508da00e5ba847ec0792f3bc373d8b4297747189ef42d1ae0"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "labs64/netlicensing-mcp", "label": 0, "pair": "32ef8148a2c08f58", "unit": "2fb2c9314a483787168a450f50430721f9527f913aa0eae07298d5334cf4f137"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "labs64/netlicensing-mcp", "label": 1, "pair": "32ef8148a2c08f58", "unit": "78dd291bb2ef83ea3a87948c0ac0027c6d667b9044093e766e2bc91bebe83936"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "langchain-ai/langchain-nvidia", "label": 0, "pair": "54586b5a24655adf", "unit": "2ece75a0240ed36a811c381841aa6d440af2b9774ddacb1d7a31a67ec267b0b2"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "langchain-ai/langchain-nvidia", "label": 1, "pair": "54586b5a24655adf", "unit": "e60f6b788a196db9bb671d621189b79a5526302a6d09c2396d67fbe0b9ab3c24"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "langflow-ai/langflow", "label": 0, "pair": "474f96e2a9b3d202", "unit": "4a5e8e8ebd7ecb0e252280af0cb38a1d7794757728b912525322ac26df5ee223"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "langflow-ai/langflow", "label": 1, "pair": "474f96e2a9b3d202", "unit": "6a888540443061d3b37c06f0cebc70cffdfa70e0aa940626e86cbff9829310fe"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "laravel/reverb", "label": 1, "pair": "a09692dc94164db8", "unit": "139cf84e79c4efd026a064b2ca0a3d7ffff87b54eae1275a07c0d69ef4981e27"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "laravel/reverb", "label": 0, "pair": "a09692dc94164db8", "unit": "828e88b656245a1d0865c79b2ed023a49e78607a4ac81d70ccd3690f4419d2d1"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "", "unit": "6cce78d7569ad8db7ca0a61d7078198f666502d26ea6c0afb03173a8a4bb16ef"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "82047c0513e43a83", "unit": "6d2dbed1ab1c9d8ac259dd4227be69641fa59d52e816a8dc547ade01d0b9b0c4"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "98ee23cd6bd31cd3", "unit": "ab4bb39be535d70cd95e9035bb958bfc7418576a6ff40a47551894dd140317bb"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "98ee23cd6bd31cd3", "unit": "b70068cbc646bd01d3e10986e002f648a74e9d798900dc59cf3039b749c95c18"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "ff168d95d8d9496e", "unit": "c6ec9478c125ed780d7013c2807f2cec1878f44253667dfed319c28cbfd17d4d"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "b1bfef46dcd3b9a5", "unit": "cd4d095d8380a2fb7ca02fc6abdbae5249cc06455db2929875533f4aa8b886dc"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "82047c0513e43a83", "unit": "d1bade217be8d345cf5d0f0442ee31924187acbb98779dada95274a77462f696"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "ff168d95d8d9496e", "unit": "d561db250b892b91feed7ae877900e1e25963444f73c7699576e010f06da73cf"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "b1bfef46dcd3b9a5", "unit": "ed78fd5a7ab30bcd6e8ddd27c706a19cfd89750de4fbf81c0b567946eb94db83"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "lepture/mistune", "label": 0, "pair": "25f75d77a669b54e", "unit": "0f2825568461562f2a915a4f16ef8a4931290fd20a9801470aeb12677e66e3f1"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "lepture/mistune", "label": 1, "pair": "25f75d77a669b54e", "unit": "bf22bb44bcff4a5baf5d550c9d3e83e5f3ca607475fb51deaf4fb6fccff3d73a"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "linuxfabrik/lib", "label": 0, "pair": "47069df6bfb3dc00", "unit": "444c1f41fc23ad656a8bf98dae58b36d0f1daa214ab030682f26111dd389e58d"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "linuxfabrik/lib", "label": 1, "pair": "47069df6bfb3dc00", "unit": "c0c8b5cca070673ea4b4bb8f08dfff86b7781780e300720c4051aded035052fe"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "cd80750447cf7af67ad809279718d98106c71705fef9ff7254728bfff4ece5b1"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "2ce605ab5ce7ad3a1fb983cd6bddf3924e5da3010d2252a0a4640aab4e5921a0"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "litedb-org/litedb", "label": 1, "pair": "b0a899116dc36b85", "unit": "510ad48af03ba4a1a2a06dd254b135311dddb62d9fe5a2e8e8b12627586e6b13"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "litedb-org/litedb", "label": 0, "pair": "b0a899116dc36b85", "unit": "b414718a8a5f79739b88db24da6ad9da8e48bcb6b5e2dc99158848cdaa226767"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "8e22ee30441b20484090659b0c97423c711f5d10219041bf3a399623fc056a07"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "0f9070df9dc334f981cb01008c260b8bff78dbdeb48ce672830ed95caa8406fb"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "5e352f797b1422b0df477bce4730a9ce33cbd0837c3d63da94e4d8801a9a4b64"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 1, "pair": "dadccb128ac0e120", "unit": "505097494da5939404ad7d5b4af2b1c4a10efe94048084492d6ce60ebaa04c97"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "dadccb128ac0e120", "unit": "84ff8ca2d98a1b5d1ca4bfaabe7d19e03a396042b228c8d18d1185a178389702"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "91e261c54bd2c3c3a9ac04e48894ecabaca5b13e7db4fbd5578f9d266098117a"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "a308dfd59bc398824e2454bf6cf6cfd5b7880e5b948a82424f14753d2b6fcdbe"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "ba24eb554458bc97314ed561a7b2f4a4e643dde1205c32de4c8fcdd8c46e575b"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "mganss/htmlsanitizer", "label": 0, "pair": "24d8f1840b67fbe5", "unit": "7ffa4a229a62266788fbf64890f81f20358b3eb8c35d3ab87d0c339d5bb4b507"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "mganss/htmlsanitizer", "label": 1, "pair": "24d8f1840b67fbe5", "unit": "d19493ed7941e1cd25a76dc50504d97398770d07ebc302dd66d83441e946c468"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 0, "pair": "9f8a60eb48d74bc8", "unit": "80fce34f2377731e3de4355c7ea33bb20fc2fc00a5f388565b23018c7a3ed008"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 1, "pair": "9f8a60eb48d74bc8", "unit": "b68e62722a327c9d5ce6774eb96527451729410768efc0b5c7ba604ec468052d"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "microsoft/prompty", "label": 0, "pair": "2c857235d0764986", "unit": "33bdfe911cd1b512e5994ee7a3e3ab2ad1f302cab29df3b011c0d825c23536a3"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "microsoft/prompty", "label": 1, "pair": "2c857235d0764986", "unit": "b5efdcb84a057f3cd9fd3a30a32239dc06c13536a4e84a66c7a17cb372050c97"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "middleapi/orpc", "label": 1, "pair": "9e84429d629c0b6b", "unit": "774cfdecac67499bb35a872931d4698d1b92ebeeb42d21790469e418889954e5"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-0", "group": "middleapi/orpc", "label": 0, "pair": "9e84429d629c0b6b", "unit": "a6c5c15ca1815f83dfb47428c4a8afd4e60fa1c027b9f33c668cb9a0d77c18af"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "mixxorz/slippers", "label": 0, "pair": "8e6b4b4e28981590", "unit": "3cce3e8f9ff3f1d261f9b96c844c1e4da07bae72d5393897b3a929be25372edc"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "mixxorz/slippers", "label": 1, "pair": "8e6b4b4e28981590", "unit": "b105689ac24ed334294c434a77bb6d56d5d598830ffe8a5d60ab3be5290b47ad"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "0a36cd5177f3b2c20b433db6bfffe5ec5b7e74cacc34a86ac581f7a0a559432a"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "e7fc97d83013c9ff", "unit": "5ab444e64297fc19065478104c7ebdabe23cc415f22adafd0cf82c98e4c5cae3"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 0, "pair": "e7fc97d83013c9ff", "unit": "6b462ea30c1afda572600a10324249223932d36a032eacafac8c837bccbdce87"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "84d6a4202a082e0d", "unit": "8175082f40d8d61d44b54579c84a808e25b89110be5d8fd82feedf7dd5e0d5ab"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "8abefca43ee4ae2c797033da31f85c17bfa7be6b35413d70b0216415c5db9a09"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "91d213946859f33d3f243c43aa8285cc6e8ecb3db1de938f32af775a70607f8e"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "a244d9f010ddb43a09e82363dfabb0e406d411fb5de08ab7d6c9ee367f2a15c3"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "d27d174d6e8e5f1682a1c42723705e89f0d18c95ecedd0567e313daa9a0aae44"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 0, "pair": "84d6a4202a082e0d", "unit": "f45608ed47fa11ca25093641f6021ec06076c18676abac9b809f31b3b7713de8"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "moquette-io/moquette", "label": 0, "pair": "55d1ce4f29dd1583", "unit": "a877ce3462da316bcdd82ce9db1d9887e887eaa83157922edb1663413edae1d6"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "moquette-io/moquette", "label": 1, "pair": "55d1ce4f29dd1583", "unit": "aee67be0ba31f99c7a24faa1bad6178c6d37361a308cf5aa10138e6ce4bbcfeb"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "naorsabag/openhop", "label": 1, "pair": "16e529fe8c5486b2", "unit": "1d21acd0171726775d3da4589b660645fcd43cc026042a2b27e0e0e8374eceb7"}
|
||||||
|
{"family": "path", "fold": "outer-4", "group": "naorsabag/openhop", "label": 0, "pair": "16e529fe8c5486b2", "unit": "f8e004b67ac98dc3392ea4115051e6bd10ce22eb638a762f0f16e1d4200bf129"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "nextauthjs/next-auth", "label": 1, "pair": "0b34438e6e0ba63d", "unit": "bf2608b00f0f784cc42e4c607f0b1ef50903283779bfb2eec91d2ec187856d14"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "nextauthjs/next-auth", "label": 0, "pair": "0b34438e6e0ba63d", "unit": "c5104cbf723fa0d55675963d8aa6802780ce3d274efe0012290fc49164c3fcc4"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "nuxt-content/mdc", "label": 0, "pair": "b0210cd069757ad2", "unit": "d1d40c8b48d039c1999ff957a015273c30652ca0c45c4eb088052991a2a92f28"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "nuxt-content/mdc", "label": 1, "pair": "b0210cd069757ad2", "unit": "e2e804652a885f640ebf341627f3279d6c965b47ee2ce792e55045ab4d1e1dee"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "nuxt-modules/og-image", "label": 1, "pair": "ef603436079b0005", "unit": "77f843b5044bb5da935df4b2620bf2abe9e7dceb3147439e883927112eaec22f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "nuxt-modules/og-image", "label": 0, "pair": "ef603436079b0005", "unit": "f2ce95f0e44e73986784b63a702ffa4035c423056cac3667a63140a7e8d795c2"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "nuxt/nuxt", "label": 0, "pair": "a023056d2704c727", "unit": "6028fe87bea019646fff2b31c1b2d7bd0a95eb9214fe437e2008bc5d2dfabb7d"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "nuxt/nuxt", "label": 1, "pair": "a023056d2704c727", "unit": "dd93069189c5f242db611e4bfba4341c22f283b65af666fc31343854358de87e"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "oauthlib/oauthlib", "label": 0, "pair": "abc80e34d5f72367", "unit": "87d4a5b81bbfdd9628109a5f820895b6fddb00eb96b0f7bbe30a8d640457bcc2"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "oauthlib/oauthlib", "label": 1, "pair": "abc80e34d5f72367", "unit": "b171c77b52637bc84c487d7f8bce9a54229f9dbd6605e5089e1e88943ced989c"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "opensearch-project/opensearch-ruby", "label": 1, "pair": "1945f28a3fc5bd55", "unit": "64c0b7ccad09192eeaf590befe0de0087d5b48a65a4f6cef41415f3f54a8f536"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "opensearch-project/opensearch-ruby", "label": 0, "pair": "1945f28a3fc5bd55", "unit": "b55cd6759d4b7861e589052aa534bd0554586db46a329ed3beed7244d579e8c4"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "openssl/openssl", "label": 0, "pair": "1c9e47835232f644", "unit": "90e220d3540b88dd9efc8e66c8ab519d4cc78805f16bed585b795c78cfbb4d58"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "openssl/openssl", "label": 1, "pair": "1c9e47835232f644", "unit": "b5a185b05235f70f6cf27e2adba919f4d8393dbdf2d99de095548a21946aeaf3"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "openwisp/openwisp-ipam", "label": 1, "pair": "9b2b34cf62a76ea3", "unit": "91a3984e47f17512db8523e04f863525c67db18631d8f11776cf61ca7221c91b"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "openwisp/openwisp-ipam", "label": 0, "pair": "9b2b34cf62a76ea3", "unit": "d91f1e14282a0c3c8da57c6935a1625e881374bd205b5baa9e4a341d9109ae21"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "oqtane/oqtane.framework", "label": 0, "pair": "a20cdf9dd208720d", "unit": "18a26df68e4b81d411cfbcc8155a7e6f382753d6e1ad95eece7ed24fa8c44324"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "oqtane/oqtane.framework", "label": 1, "pair": "a20cdf9dd208720d", "unit": "e6fb74441fb585d7ef5855317fba795ef0f0a0e6eae9970a49556ce3070c3254"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "oscal-compass/compliance-trestle", "label": 0, "pair": "b2e833b60a0fa396", "unit": "08ccacb7434e9f8b8b05c7bb0996be0609e34860d8fc418db329aaf8d7910f2f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "oscal-compass/compliance-trestle", "label": 1, "pair": "b2e833b60a0fa396", "unit": "2ca7f80f0badc049ce725718e49bd60da72d316589399126a5f4c5308cf83868"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "owen2345/camaleon-cms", "label": 1, "pair": "d6930fe7e752b7ec", "unit": "06f0341dfbf1aa9504083737df5d9be3cedfd4e3a198799d8075a29990776800"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "owen2345/camaleon-cms", "label": 0, "pair": "d6930fe7e752b7ec", "unit": "30e2cd8bd381d7568b78eaf982f562d278a6a8d942d5282466532c1441bdf0ad"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "patriksimek/vm2", "label": 0, "pair": "e9c34a6a4bf2d333", "unit": "0ae20c7334fb27889ef8551f34c65a847dffd6b37f84f722f7e8ed64fd82e95b"}
|
||||||
|
{"family": "access_control", "fold": "outer-4", "group": "patriksimek/vm2", "label": 1, "pair": "e9c34a6a4bf2d333", "unit": "59ab1924056704605b22b7c8360f40f745e7f488450be3647a632bf9a3b22699"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "pdfminer/pdfminer.six", "label": 0, "pair": "053a9e02feda3960", "unit": "34e18fe343b5c25ec582fc05ab091fd9ba160f658ba7678066c56db4991a951b"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "pdfminer/pdfminer.six", "label": 1, "pair": "053a9e02feda3960", "unit": "8fcb18a2a8654b6594cb2ec078e2a018d20f73d2b69a2e625338ffa80b3f72c3"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "perses/perses", "label": 0, "pair": "2d482951d7380c4f", "unit": "5c1a721da6f85d1d5dfc8cbe5e9809b093a09c059e5abeebed2cbbee55fbd498"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "perses/perses", "label": 1, "pair": "2d482951d7380c4f", "unit": "6cabd1c4b5aa94cafdd07c0c6a3f9abfdc03761ddc5b0a2bebc72a508d020b8b"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "", "unit": "2300c097374b9098671dcbcd42d02d08172e0f3064a8323223d0fe79a6106aab"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "pgadmin-org/pgadmin4", "label": 1, "pair": "0664148b0b192456", "unit": "d702617aaf4d19811baaa18386a3c8b0eb35cd3fd72e8d90611d34850e990858"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "0664148b0b192456", "unit": "f5ed13d811c5b16d46cbd4a3b9edb8f7d2db49acbd49d95f39f33eaab38f58f5"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "phpcsstandards/php_codesniffer", "label": 0, "pair": "fd5781ba332ab81e", "unit": "59eca69b2bd99ba282eb8056d7bc789e9c691eebf62a4b42594e34f5516833ae"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "phpcsstandards/php_codesniffer", "label": 1, "pair": "fd5781ba332ab81e", "unit": "7a7d774a6de9e4a95fe97fff822652f1f1383922f2f0a4dcb12b8e964ca330f5"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "phpoffice/phpspreadsheet", "label": 0, "pair": "366dc5ddcfe598dd", "unit": "0053d866cec1e99f8a4a8754cf8ece3c46df16e002c86703c68937d5f3a09b08"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-3", "group": "phpoffice/phpspreadsheet", "label": 1, "pair": "366dc5ddcfe598dd", "unit": "027dbc5b9245fb804a1e2ba4445b8afa180aa2c2b3293db107ffe0ca4900ebb4"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "postcss/postcss", "label": 0, "pair": "d0a34bf90e99e794", "unit": "4efbb75cc4105149522d2b2220987fd4373a8ef645c2c544546da540f14109eb"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "postcss/postcss", "label": 1, "pair": "d0a34bf90e99e794", "unit": "fd790f533103fd1a5646a9374ae895cf6e36d578fc747db2b0d0249a54a85e9f"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "powsybl/powsybl-core", "label": 0, "pair": "a697aa902d141a6a", "unit": "341e8fbb46f00167a32a2d653ba09a205794084eff65afddb70f9cfff42d697e"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "powsybl/powsybl-core", "label": 1, "pair": "a697aa902d141a6a", "unit": "e29414494b275afc15f3a028f62402161df7f6da5123c302149e23845177f7f6"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "prowler-cloud/prowler", "label": 0, "pair": "8c1fc8417190a134", "unit": "2125fb101d6108627e77f61c05cadddddfebd1cddce0e66376d9b822a2eecbb4"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "prowler-cloud/prowler", "label": 1, "pair": "8c1fc8417190a134", "unit": "ae7bbc58026fb71f91b477946a27bbe3599be90501da9237755f3cf0fb289794"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "publify/publify", "label": 0, "pair": "1972ea9f72b178b6", "unit": "3ddabb356d78e7c65ec8f88b6da8e4fcabc055ae73f2f6d92c50efddc5d4fcbe"}
|
||||||
|
{"family": "access_control", "fold": "outer-3", "group": "publify/publify", "label": 1, "pair": "1972ea9f72b178b6", "unit": "6875dd5e9c0ff8a1b693d29a109cb2e62e6ff4081df40868aca70e4770476757"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "pypa/pip", "label": 1, "pair": "2b007a2377d633dd", "unit": "bdd71a783efb4eda9a82afbd30a16bd7608153830dbc599e65c453c18d1644b2"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "pypa/pip", "label": 0, "pair": "2b007a2377d633dd", "unit": "f398ef33900624219e59de08e1cc101c43082a81c37d6d03a916cc220727ca67"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "04aa777130c02a06", "unit": "16aa43c87956c44a4b115245140eefed704095d37cec0cc6410377403cd2fa7a"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "23744137e0485a36f686f5f1b5666780fbed3ca522e8c6d35687a391ef504498"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "67565d99fe28c68f", "unit": "2692316792e6f02fb51578cf30d71bba6c73bf5420c36f709681c8eaa69807d4"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "2e36f73002c416a68d077bffb0335d82346c7507f2478353aeac005fce5a9cbc"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "04aa777130c02a06", "unit": "67721f67055eb6fed955ff22bfe4060344662eccb369431835a2116ad120c4f8"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "8fc1c96fa6faa4d3966ecc42876189b18ddab779d58f47cdf28a1155f48a6ac4"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "67565d99fe28c68f", "unit": "ab535098a8e709b022afee1da15dd1fc371d389056938ee4fa4971b2966c823a"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "d46f94235bc462360097e7a22e1d442d95f049306631489a4d8b25f647a95bbe"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "edd6dd4364442f66384d14e9317ef2f6234e32ab78327a45741b8a816b4332ee"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "fc7df6887ac719d4615934c0e8c3296728c2f6089f233158c59989027c802d72"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "rack/rack", "label": 1, "pair": "c0412509f15fe3e1", "unit": "63bda2a71ad7155ad2f59aa9086efef3fd3710dc82c57f4b32ec8bf561460239"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "rack/rack", "label": 0, "pair": "c0412509f15fe3e1", "unit": "b1c9a070c93764655786c3de5fde536c150e319efe47548614cc544b73a8ee2b"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "rails/kredis", "label": 1, "pair": "feed5f7de11289a4", "unit": "71531567b2b8609ae0d31cb177b8faea4f5fc61923e513b0154a2c941287ce2b"}
|
||||||
|
{"family": "deserialization", "fold": "outer-4", "group": "rails/kredis", "label": 0, "pair": "feed5f7de11289a4", "unit": "d6be6d397a301a07faa761163d3d9e270d71d23d3bdfd084eeafb1e62b91dd71"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "redf0x1/camofox-mcp", "label": 1, "pair": "31c259b4e9dccd82", "unit": "1f1ddb8a5b415461638056c70756d2d0e836de71e824c0e46474f20646eaa4d5"}
|
||||||
|
{"family": "access_control", "fold": "outer-1", "group": "redf0x1/camofox-mcp", "label": 0, "pair": "31c259b4e9dccd82", "unit": "a6ec568d4eef212a842dded7fdef2284524d367e5d24473ddb34e6374be9fad7"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "redisson/redisson", "label": 1, "pair": "ffd4b04a7bebfaf8", "unit": "54f7519d3d59b6d8d69a5f8f7bd78e360d3fabaae00b7587c37e388719169370"}
|
||||||
|
{"family": "deserialization", "fold": "outer-0", "group": "redisson/redisson", "label": 0, "pair": "ffd4b04a7bebfaf8", "unit": "c5e42c9ab2ee54e46700ab0b64e239b567fbc75ccaceaf6cbf20687813663332"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "redocly/redocly-cli", "label": 1, "pair": "2ffec9490ef1d482", "unit": "0ab883d3561162b75ad9d161f79c0d17521a23317f2f574c910465f288119483"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "redocly/redocly-cli", "label": 0, "pair": "2ffec9490ef1d482", "unit": "b0af86c1642c14047248f1134181be0ec3839900917dffa360c5014119f8632c"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "remix-run/react-router", "label": 1, "pair": "73f2a014c6a699ed", "unit": "44ba9d9d8e75769c1444ebf37a64941a845b4bcae678ec79d64dc84e5cc067a0"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "remix-run/react-router", "label": 0, "pair": "73f2a014c6a699ed", "unit": "b430d488c486cef26b1c661f773e7e9e4a177eeea73537a033eec98d532e4eed"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "remorses/genql", "label": 0, "pair": "738e4c22edb32172", "unit": "932d7312b2bb4b830179104bca076f587505c2d4ce881b2f04f992defba6dd9e"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "remorses/genql", "label": 1, "pair": "738e4c22edb32172", "unit": "eee5136489c3baa3653ededbf6385d0237e2bd8b5384286780646b88ed34fb91"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "remram44/taguette", "label": 1, "pair": "cb0c638282db0d37", "unit": "a49e6ad2e7c2a5ad0d68d5d69c263306b501fc0dc9382e701fb5fe680611196b"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "remram44/taguette", "label": 0, "pair": "cb0c638282db0d37", "unit": "dee28099d3b37cbd0ee14412122ecc5f8488ed6bec01863520b87706a040fba1"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "roest01/node-pdf-image", "label": 0, "pair": "09bce271e8530fbc", "unit": "35eb19b4c6176b37287e2123bb98ec4045ce6bde7cc3864dde317c024b0935d8"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "roest01/node-pdf-image", "label": 1, "pair": "09bce271e8530fbc", "unit": "dd33291cc494bbc77d44e9b13d16f171d32c9746637506f315f375e8a9ddee95"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "roomi-fields/notebooklm-mcp", "label": 1, "pair": "a856cd13668353a6", "unit": "6149fb0132b981620a90a23a9af8d4656d0d827403e63b48258443cacd0a9c81"}
|
||||||
|
{"family": "path", "fold": "outer-0", "group": "roomi-fields/notebooklm-mcp", "label": 0, "pair": "a856cd13668353a6", "unit": "b5aecdd6a87f8d57d28f54a013b1f96332460fe1bb44bff4ed28e166f18f9abe"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 1, "pair": "40d8d81a764c508f", "unit": "7deeed73c29c74ccbed29be049e109d363d41e4b5eea245ee57780b85f0e137a"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 0, "pair": "40d8d81a764c508f", "unit": "b5cdadf848e67910550666ba2a6ac67b24112e501c02dba1865766a316317a23"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "", "unit": "15dbea384b3255457a44d9382efc6944dd27f1d411d903344a5d5e5fd194b72f"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "b2ac284bd70dd684", "unit": "225890ba6694dc8f4fbab46e9c851dff8c095e906cf14db15d01ebce0891aaab"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "b6eca35de93a5be8", "unit": "42eb8427415704e060e73b5ce0943b014ddf380ca756d5ec47c558e59a79895a"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "b6eca35de93a5be8", "unit": "4c9cdcc11a18641ef41e9308712c5e8b830ec9ad5ebc3a956c8701e4f77b39c0"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "6a53939838afbdd7", "unit": "5794809e595311c86909d673a570d003aa025e4dcd6abc5641b641d935beb0b1"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "46d3ec7d49cb8ca7", "unit": "5f7a63ec650915962ee5255f5416df63ab882d6107f3b540be36d481798a8944"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "6a53939838afbdd7", "unit": "97a7018535ab511622e3ade9e53d570ffb1c80633eaa372fb76e749b3a586c8f"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "46d3ec7d49cb8ca7", "unit": "a84e9ab47f4f2b584684dbb81575758d9032fe3f2ed53bef478cb9dcfedaf677"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "b2ac284bd70dd684", "unit": "aeceb1ebc863ebbc3b69ae56c5b78d2dba005132b75cbd6af929bb0bfe25da9b"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "f42cab375ace1739", "unit": "e329d4a6ac757680bf3d292a81f1f74f8a8a9b482d1651efe9f503f6a4de8796"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "f42cab375ace1739", "unit": "e73c5aac41c0b93072223a3097fb175548ce893e0678ff3a22bc7571745a17e0"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "sebhildebrandt/systeminformation", "label": 1, "pair": "1041a18cd1aec5f8", "unit": "c301b8c362aee3fe5ea38e18f858166bc4162ad8ace44c5fb2df192afc81da2a"}
|
||||||
|
{"family": "injection", "fold": "outer-1", "group": "sebhildebrandt/systeminformation", "label": 0, "pair": "1041a18cd1aec5f8", "unit": "ca8e58ffcfd60bdc8626db8b9eb4f253c1b8b8affcb793c6d963b7a3ec91906b"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "sgabe/dsvpwa", "label": 1, "pair": "", "unit": "52354ccdbc16f209e4692d97d5237e04ed030cf4a5e71f77b5a767962b355b4a"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "sgabe/dsvpwa", "label": 1, "pair": "7d23be667536afff", "unit": "b5016a5a53409491ba805ed8718599b84250b7b6914da0434b0882b0b3578dc8"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "sgabe/dsvpwa", "label": 0, "pair": "7d23be667536afff", "unit": "f1b01102968568d3ba1b2fdc70f4ae281f433fb20d29eff2a55fc45555ddbfcd"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "shepherdwind/velocity.js", "label": 0, "pair": "f21df949f8aca4fa", "unit": "15d23ab3aea4c878dd73bbae579f3ab473e8363477a97d28164ed5e1b94cb510"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "shepherdwind/velocity.js", "label": 1, "pair": "f21df949f8aca4fa", "unit": "787bc98c651ed3db21be4c28e9c560c366ffd6ca508823d21e5d83eeaf1269db"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 0, "pair": "1199de75f426940b", "unit": "10fdc236f43a02b7e291eb8bf6ada297a2ae98086a63b171057bc7bfc37a3377"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 0, "pair": "22112c31603bbe9f", "unit": "5eb3b7523b5a196dcaacba08d68457f9a45e31503ea02745ca3a7d0542f9881c"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 1, "pair": "22112c31603bbe9f", "unit": "6bd5bdfd81b0a3374d59cfb2c1f7fa2b27887f4672f4485e77eb5234b569983f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 1, "pair": "1199de75f426940b", "unit": "c09dffd04708540845c8d1dd77a2bd989bd35f264abf652a2db55f8cb1faf7a9"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 0, "pair": "7aab025ea7b97019", "unit": "7487a34cdfdb532a5096764a47a3e565fa509605c1df0e2b105eb9c751fc5307"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 1, "pair": "7aab025ea7b97019", "unit": "9c0be2a58741f095fd0a9cdc60d4ca4039de1529699d5701dc4d0bcf7d5bf90c"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "simonh1000/angular-http-server", "label": 0, "pair": "4de5441f1776f083", "unit": "a3a35d573da308cd1d0614a26616bbd37b360d9ea416894cdeffdb59abd691d4"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "simonh1000/angular-http-server", "label": 1, "pair": "4de5441f1776f083", "unit": "dc33fc26c1bf7c67e5439ddfdad06c0faf16c796e0fe8a46b6e6a541e708676d"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "skops-dev/skops", "label": 0, "pair": "1972a9a06d7585ad", "unit": "1f472e842e894c9907536c6eb6c9c922bbb3b21c27999114588ec2a4345ab665"}
|
||||||
|
{"family": "deserialization", "fold": "outer-3", "group": "skops-dev/skops", "label": 1, "pair": "1972a9a06d7585ad", "unit": "7e38e24f1cc4d287627b7cd6cb870888865bb87fa87bba1c4fbf84eabb213121"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "smithyhq/sqladmin", "label": 1, "pair": "d76d5232a8587127", "unit": "1ec32d427822d1ddd3b0d7245f5771ddeb56c146ac8ff9959dd85760ff659a20"}
|
||||||
|
{"family": "access_control", "fold": "outer-2", "group": "smithyhq/sqladmin", "label": 0, "pair": "d76d5232a8587127", "unit": "5143b19a1a1e6b9162b80594ed7fb5c0c57ed0b8abfd793bfc71089552d57638"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "soyuka/pidusage", "label": 0, "pair": "5f6e814444b4acd8", "unit": "2e8aa07b1aa4bfc5349b0d4464f3f2a6500aa55ff765b0139e9901ed34151ffa"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "soyuka/pidusage", "label": 1, "pair": "5f6e814444b4acd8", "unit": "e26c726c08be761b22fcc6ec17fc1fa7aba4769f08957f024e5545906df62797"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "spatie/schema-org", "label": 1, "pair": "888e955fb6cfb4fd", "unit": "6f1a431ca97afc34ceec48393eb134a8928a312e6f761eb907c6b5855cac6208"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-3", "group": "spatie/schema-org", "label": 0, "pair": "888e955fb6cfb4fd", "unit": "7b62415d17569924d16238a4aa28ab1af2db6561f56be9f59d8075fd1bc4a941"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "spring-projects/spring-data-mongodb", "label": 1, "pair": "a5691395e641fd4c", "unit": "40339c123107f1231d6aa58de876cded1dfefd5b31dadc4fc86eedc6a687165f"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "spring-projects/spring-data-mongodb", "label": 0, "pair": "a5691395e641fd4c", "unit": "ea3828e2f59cc6f7b415a101532b566ecbd47ab123c3c7593c23c52f6773a2e3"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "spring-projects/spring-graphql", "label": 1, "pair": "f1ebdacbd647e39d", "unit": "79628ef3d6c9e12a94922cdca548ff77d3dbdd6b595c1fec5958fcf23b609095"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "spring-projects/spring-graphql", "label": 0, "pair": "f1ebdacbd647e39d", "unit": "f910e010c3ff2c8c17f0f80fbe89a934e4f792b601febbf318359e6dc3afc18e"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "squidfunk/mkdocs-material", "label": 0, "pair": "2dd5384f097711b5", "unit": "212fba6b5e50db045114af369a03ffbdf09ec898c10eeff3c45bfc01a5017b4c"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "squidfunk/mkdocs-material", "label": 1, "pair": "2dd5384f097711b5", "unit": "370ba3a1d0e354e882b7be92c1ebbf2e2903bdf29d442048ea3dc1f80922467f"}
|
||||||
|
{"family": "injection", "fold": "outer-4", "group": "strangerstudios/paid-memberships-pro", "label": 1, "pair": "", "unit": "ac905d12b92954263c046bd0138a682cea86cf8476336d6baa664ade28d34ac6"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 1, "pair": "218a45d869bd7e39", "unit": "309ebac468b7ab054c8396ed495770385ae3590caba43709745ba2d1db4b6d54"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 1, "pair": "", "unit": "3cc8613813dec13cb1b7b45b296688da883f0b36cd2c4fc1da6e81703fb59ab9"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 0, "pair": "218a45d869bd7e39", "unit": "9bd2331dbc260451355307e4c37bae17ad6676654c66181934e1b35fb174cc5a"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "b087ac4d0989e0b72698a9644ef4eac8073b4a15bb21730d63e4730b5cc58a60"}
|
||||||
|
{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "c5d1e8d19560c997018fdca12eb60f1799be76c33be058b88472f5a2fc6709be"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "svg/svgo", "label": 0, "pair": "c863054be488a284", "unit": "425e1d9961b79bbc011c447cd050910dd796a32b248962a5b0acc3f5f19b7345"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "svg/svgo", "label": 1, "pair": "c863054be488a284", "unit": "552925bba199119c7db5457b9d3c535bc9f6061cecfd2ab2f72cde690a536e1e"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "22befc17a75a688e2f9358d2ddf9a7b197da44cd95dbdeae5e0191b113ee54d3"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 0, "pair": "63f496d9ff53e39b", "unit": "6a98f8145776c78793b37156e9a81757841bb6de80016ab4fa575209d724db3f"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 1, "pair": "63f496d9ff53e39b", "unit": "7ac60c96c15f8517e089cc2240aedcf1fc30e54a41e35fe03c6b965bc393fbd6"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "ae39cb55ef2a87c5347e0ddc40975f142450e82de4909e93ba62a7ed0efb4134"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "thenables/thenify", "label": 1, "pair": "d1b19898f8deaa41", "unit": "3763c7b407ad471820b5f8fa25bde0ba6d91e15c07c56681de4b41add418b0b1"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "thenables/thenify", "label": 0, "pair": "d1b19898f8deaa41", "unit": "bc8b934938c30687f3d4f982392d2cfa9504a916b463ccb5c46fe8a30eede488"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "thumbor/thumbor", "label": 1, "pair": "f5f33fa6fe7b8492", "unit": "938bbf146a9037e687190bcee82c047492dffbe90efea40a49a647f81a0f1ed0"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "thumbor/thumbor", "label": 0, "pair": "f5f33fa6fe7b8492", "unit": "b90c9f7e0011685f1a8e853a9ed0ae37fd437ef6d6eb0ba17c240aec2c22b603"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "tomasnorre/crawler", "label": 0, "pair": "f74364dc8e35c8ba", "unit": "11e760e99c5c852676c9288e18a250febb3854e9047b51e8b9ecc9f3b423b6f5"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "tomasnorre/crawler", "label": 1, "pair": "f74364dc8e35c8ba", "unit": "5fc980194040f4c4d803075d4085618cc0260ae8f3454b26b9b704d30200ffea"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "tornadoweb/tornado", "label": 0, "pair": "db1c51833994ec55", "unit": "c2ce8320c778dd55434d7413844797b68e6fcabc4dc509458a9d60ca70b88c60"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "tornadoweb/tornado", "label": 1, "pair": "db1c51833994ec55", "unit": "f2d6843547af1276ae8a4dff3a7786913fa79dc69540d4fbb95c4c091e27c7f2"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "truelayer/truelayer-dotnet", "label": 1, "pair": "f4690679bf9d80aa", "unit": "1a4e24df8d08672ecde4709a77ea5c437b2198fecff8bfff373b63c9eaed6e63"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "truelayer/truelayer-dotnet", "label": 0, "pair": "f4690679bf9d80aa", "unit": "78711973ec84867671314dd7878769092b32f59860eafed50a0f6979daee76fc"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "typeorm/typeorm", "label": 1, "pair": "d6a42c18151efb48", "unit": "01ae2279707361bfc06a005335d41b85066d0a163802ca81c031f5dbded5bb46"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "typeorm/typeorm", "label": 0, "pair": "d6a42c18151efb48", "unit": "a50dc15c313a9078badea01365987c66c5cee1ad48cea65c35c8bc2cbba25e8e"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "ultimatemember/ultimatemember", "label": 0, "pair": "85c15e3fc2b370b2", "unit": "0990d8f41cb07c1cf217812e51a7025381de4ba68d1be3799e26241d81dcea67"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "ultimatemember/ultimatemember", "label": 1, "pair": "85c15e3fc2b370b2", "unit": "9da1d63f0c4da270f66eb87bf9c342ace883a0aef78c4643e254159b5da8fce3"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "unjs/ufo", "label": 1, "pair": "c7f4f4ebebfae831", "unit": "b5cbc1490bca76679adfbe5e4d9f8067d27a226d595ba401b551b00c3ce4d45a"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-1", "group": "unjs/ufo", "label": 0, "pair": "c7f4f4ebebfae831", "unit": "ba2c1808eebdbe3f54c488b09bcb3d69e521916d1a3a0b5b14e02d5bc88bf016"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "vercel/next.js", "label": 1, "pair": "1507c8e00ed4b859", "unit": "620e9b5ed143c0bcec178db359fa601acb1d3a134b3e85c2e838b7dcf3892d85"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "vercel/next.js", "label": 0, "pair": "1507c8e00ed4b859", "unit": "6f5ce4008dac0471260310c60e1e2a9f76a7e4eec6ebd039835f0488cf3b19aa"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "viewcomponent/view_component", "label": 0, "pair": "8f5a6abedd742bbe", "unit": "71e3ced643a244100cc3fde234f5b4b27655214c7ae0ff67ab61e3e48a8fdf78"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-1", "group": "viewcomponent/view_component", "label": 1, "pair": "8f5a6abedd742bbe", "unit": "7c8cbfae87127042f9f43621345b4f7958bde27a3ff9db76254063e80735f3ce"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "vincentmorneau/apex-publish-static-files", "label": 1, "pair": "c9538342f2d7efca", "unit": "2b8db57d65ef1d67c457ff5a336e8a542d03387395d69d5e16815c4f00f40723"}
|
||||||
|
{"family": "injection", "fold": "outer-3", "group": "vincentmorneau/apex-publish-static-files", "label": 0, "pair": "c9538342f2d7efca", "unit": "e2ed113dc21ad5f7caa71ba4a5ece8f8f900a97fd0d00dc85f7ac79ff5bed19a"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "vitest-dev/vitest", "label": 0, "pair": "494dd5d5e5cdf4b8", "unit": "3b8011f94ec721f68dd29afc6c8b01117c86a8b0c31f3650a6ce0a8d7c0739ff"}
|
||||||
|
{"family": "path", "fold": "outer-2", "group": "vitest-dev/vitest", "label": 1, "pair": "494dd5d5e5cdf4b8", "unit": "f7e2280574e9ae51daa64d8adb2955407ff6d1e156f0e2854b6c047e0a733c78"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "vllm-project/vllm", "label": 1, "pair": "51eae488182a123a", "unit": "20b039866ba796c34efa0e92468fb49a314ad7fe2914d9ff8b823e3e35ab26f9"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-4", "group": "vllm-project/vllm", "label": 0, "pair": "51eae488182a123a", "unit": "2ab3fef05def768ef7b4165bcedcf7d200282dba78a32e19b90a0df3aee7e3e5"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "vuejs/vue-cli", "label": 1, "pair": "6f5a650865e08571", "unit": "e2051cc42e4f8b026c7807f966d492367310c90326e60fef1b0ee8c509c805a9"}
|
||||||
|
{"family": "untrusted_destination", "fold": "outer-2", "group": "vuejs/vue-cli", "label": 0, "pair": "6f5a650865e08571", "unit": "f5364d68ffe80b578e788f37f08784b0162826393b49b4a3affd70d7cc4c9e01"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "vvo/node-whereis", "label": 1, "pair": "8acd426c5771b37c", "unit": "983e146a60e061254b15dc8dd573cbba57f8dd0358f23ea1b9c313ef69069af5"}
|
||||||
|
{"family": "injection", "fold": "outer-2", "group": "vvo/node-whereis", "label": 0, "pair": "8acd426c5771b37c", "unit": "f80dfec45231e4778078cb10f00128ed445177cac0a062fc2df5ced5c0171f21"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "waterfutures/epyt-flow", "label": 0, "pair": "9b84800ce5c395cb", "unit": "9b7dfdceae03fd86ad3260a03f85bcdcff8f32f10c265d23743ff8dc98eeefb8"}
|
||||||
|
{"family": "deserialization", "fold": "outer-2", "group": "waterfutures/epyt-flow", "label": 1, "pair": "9b84800ce5c395cb", "unit": "b99dda150c0da3defa5f4c527ef6c402f5589dfb2f2f03f3e5e424910eb48c53"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "webpack/webpack-dev-middleware", "label": 1, "pair": "ad64e8fac4ac3ee0", "unit": "36dfd01915a553d86576e8ecd3b737b8e2b297aa542f58cc83b448060e74398a"}
|
||||||
|
{"family": "path", "fold": "outer-1", "group": "webpack/webpack-dev-middleware", "label": 0, "pair": "ad64e8fac4ac3ee0", "unit": "c2b6d40f130d35bbfc16fab9411c664fa06321c59fb1d901a1987a69cb09c72e"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "wintercms/winter", "label": 0, "pair": "d48c08b19f560848", "unit": "03d02c9e25eec0f380b125279fce35e8867065bf6e1ec75f9f39729ce6efdc77"}
|
||||||
|
{"family": "access_control", "fold": "outer-0", "group": "wintercms/winter", "label": 1, "pair": "d48c08b19f560848", "unit": "abbea69b7d0fa68c3ace442c25597e128cde354b417787b5913fd10d666c0d19"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "xdan/jodit", "label": 0, "pair": "bfc8188bbb673e3b", "unit": "a7cff0da55c0cc1d5c838e11fc6ce6c4b22c54b9f8e101688b218d6928eca504"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-2", "group": "xdan/jodit", "label": 1, "pair": "bfc8188bbb673e3b", "unit": "f741c56815230b9c30e2c8ef8952b895e9480e4780a89ff2cf6daf60d1fce7b6"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "xyproto/algernon", "label": 0, "pair": "3844ed3d703d7ef4", "unit": "445efda074e4d9f6c1634225eb08a3c3eb1257311c995ce891df2c791a6f5c8b"}
|
||||||
|
{"family": "output_encoding", "fold": "outer-4", "group": "xyproto/algernon", "label": 1, "pair": "3844ed3d703d7ef4", "unit": "8ac27b08549353ada819caaf99a5180d2f2f515b03aa8cd99a5d37b527ea3a5c"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 1, "pair": "a63eae7d621a985e", "unit": "04ca0b583b09c3b4f67bd2496a4607ecb134d24b7bdaf8960fa151e40ebcc447"}
|
||||||
|
{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 0, "pair": "a63eae7d621a985e", "unit": "ecfa2a67726ca9d3e2621a4d8be60145095b76aa18ecf0268212341d63752871"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "zumba/json-serializer", "label": 0, "pair": "0d9f529b7f0084b6", "unit": "35aa369236f510634b3aa7b54b04e9da0f80877e515786a888e760f8fe33c524"}
|
||||||
|
{"family": "deserialization", "fold": "outer-1", "group": "zumba/json-serializer", "label": 1, "pair": "0d9f529b7f0084b6", "unit": "ef4f8613c0914eb815504e00cd6ee05df38352fd3b27826798e6623594017042"}
|
||||||
@@ -60,18 +60,24 @@ def add_commands(learn_sub: Any) -> None:
|
|||||||
("register", "record the manifest's and the units file's sha256 as the experiment row (both committed at HEAD)"),
|
("register", "record the manifest's and the units file's sha256 as the experiment row (both committed at HEAD)"),
|
||||||
("units", "freeze the units of a program experiment from the memory (no model): the families' evaluation candidates"),
|
("units", "freeze the units of a program experiment from the memory (no model): the families' evaluation candidates"),
|
||||||
("run", "run every arm on every unit, paired; arm A replays from the response cache when the manifest says so"),
|
("run", "run every arm on every unit, paired; arm A replays from the response cache when the manifest says so"),
|
||||||
|
("fill", "fill a rule experiment's response cache under a separate budget, or estimate with --dry-run"),
|
||||||
("analyse", "paired cluster bootstrap, exact McNemar, the two looks and the adoption verdict (counts and intervals only)"),
|
("analyse", "paired cluster bootstrap, exact McNemar, the two looks and the adoption verdict (counts and intervals only)"),
|
||||||
):
|
):
|
||||||
sub = experiment_sub.add_parser(name, help=text)
|
sub = experiment_sub.add_parser(name, help=text)
|
||||||
sub.add_argument("manifest", type=Path, help="plane/experiments/<id>.yaml")
|
sub.add_argument("manifest", type=Path, help="plane/experiments/<id>.yaml")
|
||||||
sub.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)")
|
sub.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)")
|
||||||
if name in ("units", "run"):
|
if name in ("units", "run", "fill"):
|
||||||
sub.add_argument("--profile", choices=("static", "plane"), default="static")
|
sub.add_argument("--profile", choices=("static", "plane"), default="static")
|
||||||
if name == "run":
|
if name in ("run", "fill"):
|
||||||
sub.add_argument(
|
sub.add_argument(
|
||||||
"--manifest-model", type=Path, default=Path("plane/models/deepseek-flash.yaml"), help="the chat Model (prices)"
|
"--manifest-model", type=Path, default=Path("plane/models/deepseek-flash.yaml"), help="the chat Model (prices)"
|
||||||
)
|
)
|
||||||
|
if name == "run":
|
||||||
sub.add_argument("--no-early-stop", action="store_true", help="take no first look during the run")
|
sub.add_argument("--no-early-stop", action="store_true", help="take no first look during the run")
|
||||||
|
if name == "fill":
|
||||||
|
sub.add_argument("--budget-usd", type=float, help="required for paid fill; includes the smoke call")
|
||||||
|
sub.add_argument("--dry-run", action="store_true", help="inspect and estimate only; no calls or writes, including --out")
|
||||||
|
sub.add_argument("--out", type=Path, help="also write the fill report as JSON (ignored for --dry-run)")
|
||||||
if name in ("run", "analyse"):
|
if name in ("run", "analyse"):
|
||||||
sub.add_argument("--resamples", type=int, help="bootstrap resamples (default: the manifest's)")
|
sub.add_argument("--resamples", type=int, help="bootstrap resamples (default: the manifest's)")
|
||||||
sub.add_argument("--out", type=Path, help="write the counts-only report here (analyse --record: benchmarks/experiments/<id>/)")
|
sub.add_argument("--out", type=Path, help="write the counts-only report here (analyse --record: benchmarks/experiments/<id>/)")
|
||||||
@@ -154,6 +160,33 @@ def _experiment(args: argparse.Namespace) -> int:
|
|||||||
data = store.get_blob("excerpts", sha)
|
data = store.get_blob("excerpts", sha)
|
||||||
return data.decode("utf-8") if data is not None else None
|
return data.decode("utf-8") if data is not None else None
|
||||||
|
|
||||||
|
if args.experiment_command == "fill":
|
||||||
|
from ..tool_hunter import ChatClient
|
||||||
|
from .fill import fill_rule
|
||||||
|
|
||||||
|
def resolve_fill_client() -> ChatClient:
|
||||||
|
from ..config import load_config
|
||||||
|
from ..model.endpoint import resolve_chat_endpoint
|
||||||
|
|
||||||
|
resolved = resolve_chat_endpoint(load_config(None))
|
||||||
|
if resolved is None:
|
||||||
|
raise ex.ExperimentError("no chat endpoint configured; nothing was spent")
|
||||||
|
return resolved[0]
|
||||||
|
|
||||||
|
report = fill_rule(
|
||||||
|
store,
|
||||||
|
manifest,
|
||||||
|
memory,
|
||||||
|
parameters,
|
||||||
|
excerpt_text,
|
||||||
|
vectors,
|
||||||
|
client_factory=resolve_fill_client,
|
||||||
|
budget_usd=args.budget_usd,
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
)
|
||||||
|
_write(report, None if args.dry_run else args.out)
|
||||||
|
return {"done": 0, "dry_run": 0, "unfinished": 3, "failed": 1}[report["status"]]
|
||||||
|
|
||||||
if manifest.kind == "rule": # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved
|
if manifest.kind == "rule": # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved
|
||||||
caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store)
|
caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store)
|
||||||
summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors)
|
summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors)
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ import math
|
|||||||
import random
|
import random
|
||||||
import subprocess
|
import subprocess
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||||
|
from contextlib import suppress
|
||||||
from dataclasses import asdict, dataclass, field, fields, replace
|
from dataclasses import asdict, dataclass, field, fields, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -340,8 +341,22 @@ def registration_id(experiment_id: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None:
|
def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None:
|
||||||
|
"""Return the authoritative registration, preferring a set units digest over pending rows.
|
||||||
|
|
||||||
|
Within that group, use the greatest ``created`` value when all values are nonempty and mutually
|
||||||
|
comparable (strings compare lexically). Otherwise, or on a tie, use the lexically greatest ``pin``,
|
||||||
|
then the last row in store order (key/id order for MemoryStore, insertion order for an append-only store).
|
||||||
|
"""
|
||||||
rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK]
|
rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK]
|
||||||
return rows[0] if rows else None
|
if not rows:
|
||||||
|
return None
|
||||||
|
rows = [r for r in rows if r.get("units_digest")] or rows
|
||||||
|
rows = sorted(rows, key=lambda r: str(r.get("pin", "")))
|
||||||
|
if all(r.get("created") for r in rows):
|
||||||
|
# Incomparable timestamps retain the deterministic pin/store ordering.
|
||||||
|
with suppress(TypeError):
|
||||||
|
rows = sorted(rows, key=lambda r: r["created"])
|
||||||
|
return rows[-1]
|
||||||
|
|
||||||
|
|
||||||
def register(
|
def register(
|
||||||
|
|||||||
@@ -0,0 +1,207 @@
|
|||||||
|
"""Fill rule-experiment responses without writing scores or changing the frozen population.
|
||||||
|
|
||||||
|
Estimates use the family's mean billed cached response cost when usage is available.
|
||||||
|
Otherwise they use Prompt.estimated_tokens and 300 output tokens. Unknown answers may require one retry;
|
||||||
|
dry-run counts those separately as conditional requests, never fabricating an answer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
from collections.abc import Callable, Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from ..model.endpoint import DeepSeekChatClient
|
||||||
|
from ..plane.budget import BudgetExhausted, MeteredClient, cost_of, prices_from
|
||||||
|
from ..plane.memory import MemoryStore
|
||||||
|
from ..tool_hunter import ChatClient, ChatResponse
|
||||||
|
from . import experiments as ex
|
||||||
|
from .examples import Example
|
||||||
|
from .program import Caller, Program, Prompt, parse_answer, request_key
|
||||||
|
|
||||||
|
DEFAULT_OUTPUT_TOKENS = 300
|
||||||
|
DEFAULT_MARGIN_USD = 0.01
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Request:
|
||||||
|
family: str
|
||||||
|
prompt: Prompt
|
||||||
|
temperature: float
|
||||||
|
sample: str
|
||||||
|
|
||||||
|
def key(self, caller: Caller) -> str:
|
||||||
|
return request_key(caller.model, caller.params_digest, self.prompt.messages, self.temperature, self.sample, True)
|
||||||
|
|
||||||
|
def ask(self, caller: Caller) -> str:
|
||||||
|
return caller.ask(self.prompt.messages, temperature=self.temperature, sample=self.sample)
|
||||||
|
|
||||||
|
def retry(self) -> Request:
|
||||||
|
return Request(self.family, self.prompt, 0.0, f"{self.sample}:retry")
|
||||||
|
|
||||||
|
|
||||||
|
class FillMeter(MeteredClient):
|
||||||
|
"""Reserve a whole call before admission, including the first (smoke) call.
|
||||||
|
|
||||||
|
The fixed cent is a floor, never replaced by a smaller observed cost. For the
|
||||||
|
production adapter, UTF-8 bytes conservatively bound input tokens (plus 256 framing
|
||||||
|
tokens per message), and the provider enforces the output limit. Injected clients
|
||||||
|
must honour the same token/usage contract. No hidden transport retries are allowed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, client: ChatClient, parameters: Mapping[str, Any], budget: float, output_limit: int) -> None:
|
||||||
|
super().__init__(client, prices=parameters, budget_usd=budget)
|
||||||
|
self.limit = budget
|
||||||
|
self.output_limit = output_limit
|
||||||
|
self.largest_call_usd = 0.0
|
||||||
|
|
||||||
|
def complete(self, **kwargs: Any) -> ChatResponse:
|
||||||
|
assert self.prices is not None
|
||||||
|
messages = kwargs["messages"]
|
||||||
|
input_bound = sum(len(str(m["content"]).encode("utf-8")) + 256 for m in messages)
|
||||||
|
bound = (
|
||||||
|
input_bound * max(self.prices.input_per_m, self.prices.cache_hit_per_m) + self.output_limit * self.prices.output_per_m
|
||||||
|
) / 1_000_000
|
||||||
|
reserve = max(DEFAULT_MARGIN_USD, self.largest_call_usd, bound)
|
||||||
|
self.budget_usd = max(0.0, self.limit - reserve)
|
||||||
|
before = self.usd or 0.0
|
||||||
|
tokens_before = sum(self.usage.values())
|
||||||
|
try:
|
||||||
|
response = super().complete(**kwargs)
|
||||||
|
except BudgetExhausted:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
self.calls += 1 # attempted provider calls count even when no response was returned
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self.largest_call_usd = max(self.largest_call_usd, (self.usd or 0.0) - before)
|
||||||
|
if sum(self.usage.values()) <= tokens_before:
|
||||||
|
raise RuntimeError("provider returned no metered token usage")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def fill_rule(
|
||||||
|
store: MemoryStore,
|
||||||
|
manifest: ex.Manifest,
|
||||||
|
memory: Sequence[Example],
|
||||||
|
parameters: Mapping[str, Any],
|
||||||
|
excerpt_text: Callable[[str], str | None],
|
||||||
|
vectors: Mapping[str, Sequence[float]] | None = None,
|
||||||
|
*,
|
||||||
|
client: ChatClient | None = None,
|
||||||
|
client_factory: Callable[[], ChatClient] | None = None,
|
||||||
|
budget_usd: float | None = None,
|
||||||
|
dry_run: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Inspect or fill every frozen unit. The first cache miss is the smoke call.
|
||||||
|
|
||||||
|
A successful smoke is retained in the cache; any provider error stops immediately
|
||||||
|
and is reported without its message, which may contain credentials. A factory is
|
||||||
|
resolved only after validation, and only when a paid request is actually missing.
|
||||||
|
"""
|
||||||
|
from .evaluate import candidate_from_example
|
||||||
|
|
||||||
|
if manifest.kind != "rule":
|
||||||
|
raise ex.ExperimentError("experiment fill supports kind: rule only")
|
||||||
|
if not dry_run and (budget_usd is None or not math.isfinite(budget_usd) or budget_usd < 0):
|
||||||
|
raise ex.ExperimentError("--budget-usd must be a finite non-negative number (or use --dry-run)")
|
||||||
|
prices = prices_from(parameters)
|
||||||
|
if prices is None or any(not math.isfinite(p) or p < 0 for p in (prices.input_per_m, prices.output_per_m, prices.cache_hit_per_m)):
|
||||||
|
raise ex.ExperimentError("fill requires finite, non-negative Model prices")
|
||||||
|
ex.check_registered(store, manifest)
|
||||||
|
units, folds, _ = ex._checked_units(manifest, memory)
|
||||||
|
programs = {f: Program(ex.arm_spec(manifest, manifest.arms["A"], f, store), memory, excerpt_text, vectors) for f in manifest.families}
|
||||||
|
models = {p.spec.model for p in programs.values()}
|
||||||
|
if len(models) != 1:
|
||||||
|
raise ex.ExperimentError("the rule programs must use the same model")
|
||||||
|
caller = Caller(None, next(iter(models)), parameters, store=store)
|
||||||
|
index = {e.id: e for e in memory}
|
||||||
|
candidate_of = candidate_from_example(excerpt_text, vectors)
|
||||||
|
requests: list[Request] = []
|
||||||
|
for unit in sorted(units, key=lambda u: (u.group, u.family, u.unit)):
|
||||||
|
candidate = candidate_of(index[unit.unit])
|
||||||
|
if candidate is None:
|
||||||
|
raise ex.ExperimentError(f"{manifest.id}: unit {unit.unit[:12]} has no excerpt in the store")
|
||||||
|
for prompt, _, temperature, sample in programs[unit.family].prepare_samples(
|
||||||
|
candidate, folds[unit.fold], seed=int(manifest.folds.get("seed", 0))
|
||||||
|
):
|
||||||
|
requests.append(Request(unit.family, prompt, temperature, sample))
|
||||||
|
|
||||||
|
# Request counts are distinct cache keys per family, even when multiple units share one.
|
||||||
|
def inventory() -> dict[str, dict[str, Request]]:
|
||||||
|
result: dict[str, dict[str, Request]] = {f: {} for f in manifest.families}
|
||||||
|
for req in requests:
|
||||||
|
result[req.family][req.key(caller)] = req
|
||||||
|
entry = caller._cached(req.key(caller))
|
||||||
|
if entry is not None and parse_answer(entry.get("content"), req.prompt.valid_lines) is None:
|
||||||
|
retry = req.retry()
|
||||||
|
result[req.family][retry.key(caller)] = retry
|
||||||
|
return result
|
||||||
|
|
||||||
|
initial = inventory()
|
||||||
|
initially_cached = {key for cells in initial.values() for key in cells if caller._cached(key) is not None}
|
||||||
|
families: dict[str, Any] = {}
|
||||||
|
for family, cells in initial.items():
|
||||||
|
usages = [
|
||||||
|
usage
|
||||||
|
for key in cells
|
||||||
|
if (entry := caller._cached(key)) is not None and isinstance(usage := entry.get("usage"), Mapping) and usage
|
||||||
|
]
|
||||||
|
outputs = [float(usage["completion_tokens"]) for usage in usages if "completion_tokens" in usage]
|
||||||
|
mean_output = sum(outputs) / len(outputs) if outputs else DEFAULT_OUTPUT_TOKENS
|
||||||
|
missing = [req for key, req in cells.items() if key not in initially_cached]
|
||||||
|
families[family] = {
|
||||||
|
"units": sum(u.family == family for u in units),
|
||||||
|
"requests": len(cells),
|
||||||
|
"cached": len(cells) - len(missing),
|
||||||
|
"missing": len(missing),
|
||||||
|
"filled": 0,
|
||||||
|
"still_missing": len(missing),
|
||||||
|
"estimate_method": "measured" if usages else "estimated",
|
||||||
|
"estimated_usd": len(missing) * sum(cost_of(usage, prices) for usage in usages) / len(usages)
|
||||||
|
if usages
|
||||||
|
else sum((r.prompt.estimated_tokens * prices.input_per_m + mean_output * prices.output_per_m) / 1_000_000 for r in missing),
|
||||||
|
"estimated_output_tokens": mean_output,
|
||||||
|
"conditional_retries": sum(not r.sample.endswith(":retry") for r in missing),
|
||||||
|
}
|
||||||
|
report: dict[str, Any] = {
|
||||||
|
"experiment": manifest.id,
|
||||||
|
"status": "dry_run" if dry_run else "done",
|
||||||
|
"families": families,
|
||||||
|
"client_calls": 0,
|
||||||
|
"usd": 0.0,
|
||||||
|
"reason": "",
|
||||||
|
}
|
||||||
|
if dry_run or not any(c["missing"] for c in families.values()):
|
||||||
|
return report
|
||||||
|
meter: FillMeter | None = None
|
||||||
|
try:
|
||||||
|
if client is None and client_factory is not None:
|
||||||
|
client = client_factory()
|
||||||
|
if client is None:
|
||||||
|
raise ex.ExperimentError("no chat endpoint configured; nothing was spent")
|
||||||
|
output_limit = max(p.spec.max_output_tokens for p in programs.values())
|
||||||
|
if isinstance(client, DeepSeekChatClient):
|
||||||
|
client = client.bounded(output_limit)
|
||||||
|
assert budget_usd is not None
|
||||||
|
meter = FillMeter(client, parameters, budget_usd, output_limit)
|
||||||
|
caller.client = meter
|
||||||
|
for req in requests:
|
||||||
|
content = req.ask(caller)
|
||||||
|
if parse_answer(content, req.prompt.valid_lines) is None:
|
||||||
|
req.retry().ask(caller)
|
||||||
|
except BudgetExhausted:
|
||||||
|
report.update(status="unfinished", reason="budget exhausted; resume with a new fill budget")
|
||||||
|
except ex.ExperimentError:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
# Never echo provider exceptions: response bodies and URLs may contain secrets.
|
||||||
|
report.update(status="failed", reason="provider or cache failure; fill stopped")
|
||||||
|
final = inventory()
|
||||||
|
for family, cells in final.items():
|
||||||
|
cached = {key for key in cells if caller._cached(key) is not None}
|
||||||
|
families[family].update(requests=len(cells), filled=len(cached - initially_cached), still_missing=len(cells) - len(cached))
|
||||||
|
if meter is not None:
|
||||||
|
report.update(client_calls=meter.calls, usd=meter.usd)
|
||||||
|
return report
|
||||||
@@ -506,20 +506,24 @@ class Program:
|
|||||||
out.append((prompt, got))
|
out.append((prompt, got))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
def decide(self, candidate: Candidate, fold: Fold, caller: Caller, *, seed: int = 0) -> Decision:
|
def prepare_samples(self, candidate: Candidate, fold: Fold, *, seed: int = 0) -> list[tuple[Prompt, Retrieval, float, str]]:
|
||||||
|
"""The exact prompts, temperatures and sample labels shared by execution and cache planning."""
|
||||||
if self.spec.sampling == RETRIEVAL_ENSEMBLE:
|
if self.spec.sampling == RETRIEVAL_ENSEMBLE:
|
||||||
prepared = self.prepare_ensemble(candidate, fold, seed=seed)
|
prepared = self.prepare_ensemble(candidate, fold, seed=seed)
|
||||||
answers = [
|
return [(prompt, got, TEMPERATURES[0], str(j)) for j, (prompt, got) in enumerate(prepared)]
|
||||||
sample_answer(prompt, caller, temperature=TEMPERATURES[0], sample=str(j), families=self.families)
|
|
||||||
for j, (prompt, _) in enumerate(prepared)
|
|
||||||
]
|
|
||||||
got = prepared[0][1]
|
|
||||||
return replace(aggregate(answers), retrieval=got.mode, neighbours=got.neighbours)
|
|
||||||
if self.spec.sampling != SAMPLING[0]:
|
if self.spec.sampling != SAMPLING[0]:
|
||||||
raise ValueError(f"unknown sampling {self.spec.sampling!r} (one of {', '.join(SAMPLING)})")
|
raise ValueError(f"unknown sampling {self.spec.sampling!r} (one of {', '.join(SAMPLING)})")
|
||||||
prompt, got = self.prepare(candidate, fold, seed=seed)
|
prompt, got = self.prepare(candidate, fold, seed=seed)
|
||||||
decision = classify(prompt, caller, self.spec.k, families=self.families)
|
return [(prompt, got, TEMPERATURES[0] if j == 0 else TEMPERATURES[1], str(j)) for j in range(self.spec.k)]
|
||||||
return replace(decision, retrieval=got.mode, neighbours=got.neighbours)
|
|
||||||
|
def decide(self, candidate: Candidate, fold: Fold, caller: Caller, *, seed: int = 0) -> Decision:
|
||||||
|
prepared = self.prepare_samples(candidate, fold, seed=seed)
|
||||||
|
answers = [
|
||||||
|
sample_answer(prompt, caller, temperature=temperature, sample=sample, families=self.families)
|
||||||
|
for prompt, _, temperature, sample in prepared
|
||||||
|
]
|
||||||
|
got = prepared[0][1]
|
||||||
|
return replace(aggregate(answers), retrieval=got.mode, neighbours=got.neighbours)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@@ -96,6 +96,21 @@ class DeepSeekChatClient:
|
|||||||
self._disable_thinking = disable_thinking
|
self._disable_thinking = disable_thinking
|
||||||
self.endpoint = endpoint
|
self.endpoint = endpoint
|
||||||
self.usage: list[dict[str, object]] = [] # one entry per call, for the round's cost meter
|
self.usage: list[dict[str, object]] = [] # one entry per call, for the round's cost meter
|
||||||
|
self._output_limit: int | None = None
|
||||||
|
|
||||||
|
def bounded(self, max_output_tokens: int) -> DeepSeekChatClient:
|
||||||
|
"""A single-attempt, output-capped copy for callers reserving a hard per-call budget.
|
||||||
|
|
||||||
|
Empty JSON retries belong to the caller's meter, and transport failures must
|
||||||
|
stop that caller before another potentially billable attempt.
|
||||||
|
"""
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
|
if max_output_tokens <= 0:
|
||||||
|
raise ValueError("max_output_tokens must be positive")
|
||||||
|
client = DeepSeekChatClient(replace(self._client, max_attempts=1), disable_thinking=self._disable_thinking, endpoint=self.endpoint)
|
||||||
|
client._output_limit = max_output_tokens
|
||||||
|
return client
|
||||||
|
|
||||||
def complete(
|
def complete(
|
||||||
self,
|
self,
|
||||||
@@ -117,7 +132,7 @@ class DeepSeekChatClient:
|
|||||||
logprobs=logprobs,
|
logprobs=logprobs,
|
||||||
temperature=temperature,
|
temperature=temperature,
|
||||||
)
|
)
|
||||||
if json_object and not (response.content or "").strip():
|
if self._output_limit is None and json_object and not (response.content or "").strip():
|
||||||
response = self._call(
|
response = self._call(
|
||||||
model=model,
|
model=model,
|
||||||
messages=messages,
|
messages=messages,
|
||||||
@@ -147,6 +162,8 @@ class DeepSeekChatClient:
|
|||||||
temperature: float | None = None,
|
temperature: float | None = None,
|
||||||
) -> ChatResponse:
|
) -> ChatResponse:
|
||||||
extra: dict[str, object] = {}
|
extra: dict[str, object] = {}
|
||||||
|
if self._output_limit is not None:
|
||||||
|
extra["max_tokens"] = self._output_limit
|
||||||
if self._disable_thinking:
|
if self._disable_thinking:
|
||||||
extra["thinking"] = {"type": "disabled"} # otherwise temperature is silently ignored
|
extra["thinking"] = {"type": "disabled"} # otherwise temperature is silently ignored
|
||||||
if temperature is not None:
|
if temperature is not None:
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
"""A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and
|
"""A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and
|
||||||
pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule,
|
pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule,
|
||||||
arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores
|
arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores
|
||||||
are ``FileStore`` under ``tmp_path``."""
|
are in-memory fakes or ``FileStore`` under ``tmp_path``."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from learn_fixtures import ScriptedChat, corpus, x_record
|
from learn_fixtures import ScriptedChat, corpus, x_record
|
||||||
@@ -18,7 +19,7 @@ from openultrasast.learn.compile import canonical, program_id
|
|||||||
from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec
|
from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec
|
||||||
from openultrasast.learn.retrieve import Target
|
from openultrasast.learn.retrieve import Target
|
||||||
from openultrasast.plane.budget import MeteredClient
|
from openultrasast.plane.budget import MeteredClient
|
||||||
from openultrasast.plane.memory import FileStore
|
from openultrasast.plane.memory import FileStore, MemoryStore, Record
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32}
|
PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32}
|
||||||
@@ -170,6 +171,91 @@ def test_register_refuses_uncommitted_and_modified_manifests(tmp_path: Path) ->
|
|||||||
assert len(store.rows(kind="experiment")) == 1
|
assert len(store.rows(kind="experiment")) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def registration_store() -> Mock:
|
||||||
|
records: list[Record] = []
|
||||||
|
store = Mock(spec=MemoryStore)
|
||||||
|
store.put_row.side_effect = lambda row: records.append(Record(dict(row), "memory", None))
|
||||||
|
store.rows.side_effect = lambda *, repo, kind: [r for r in records if r.row["repo"] == repo and r.row["kind"] == kind]
|
||||||
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_prefers_frozen_over_pending(tmp_path: Path, registration_store: Mock) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
pending = ex.register(registration_store, manifest, commit_of=committed, created="2026-10-03")
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
frozen = ex.register(registration_store, manifest, commit_of=lambda _: PIN, created="2026-10-02")
|
||||||
|
assert pending["units_status"] == "pending"
|
||||||
|
assert len(registration_store.rows(repo=ex.experiment_repo(manifest.id), kind=ex.EXPERIMENT_KIND)) == 2
|
||||||
|
assert ex.registered(registration_store, manifest.id) == frozen
|
||||||
|
assert ex.check_registered(registration_store, manifest, need_units=True) == frozen
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_frozen_only(tmp_path: Path, registration_store: Mock) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
frozen = ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
assert ex.registered(registration_store, manifest.id) == frozen
|
||||||
|
assert ex.check_registered(registration_store, manifest, need_units=True) == frozen
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_pending_only_refuses_units(tmp_path: Path, registration_store: Mock) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
pending = ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
assert ex.registered(registration_store, manifest.id) == pending
|
||||||
|
with pytest.raises(ex.ExperimentError, match=r"the units file was not registered \(pending\)"):
|
||||||
|
ex.check_registered(registration_store, manifest, need_units=True)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("freeze", [False, True])
|
||||||
|
def test_register_still_refuses_different_manifest_digest(tmp_path: Path, registration_store: Mock, freeze: bool) -> None:
|
||||||
|
path = write_manifest(tmp_path, "0" * 64)
|
||||||
|
manifest = ex.load_manifest(path)
|
||||||
|
ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
if freeze:
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
ex.register(registration_store, manifest, commit_of=lambda _: PIN)
|
||||||
|
path.write_text(path.read_text().replace("mde: 0.1", "mde: 0.2"))
|
||||||
|
with pytest.raises(ex.ExperimentError, match="A changed manifest is a new experiment"):
|
||||||
|
ex.register(registration_store, ex.load_manifest(path), commit_of=committed)
|
||||||
|
assert registration_store.put_row.call_count == 1 + freeze
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("first_created", "second_created", "first_pin", "second_pin", "expected"),
|
||||||
|
[
|
||||||
|
("2026-10-03", "2026-10-02", PIN, COMMIT, 0),
|
||||||
|
("2026-10-02", "2026-10-03", COMMIT, PIN, 1),
|
||||||
|
("2026-10-02", "2026-10-02", PIN, COMMIT, 1),
|
||||||
|
("", "2026-10-02", COMMIT, PIN, 0),
|
||||||
|
(None, None, PIN, COMMIT, 1),
|
||||||
|
("2026-10-03", 123, PIN, COMMIT, 1),
|
||||||
|
("", "", PIN, PIN, 1),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_registered_orders_frozen_rows(
|
||||||
|
tmp_path: Path,
|
||||||
|
registration_store: Mock,
|
||||||
|
first_created: object,
|
||||||
|
second_created: object,
|
||||||
|
first_pin: str,
|
||||||
|
second_pin: str,
|
||||||
|
expected: int,
|
||||||
|
) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
frozen = ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
rows = [dict(frozen, pin=pin, created=created) for pin, created in ((first_pin, first_created), (second_pin, second_created))]
|
||||||
|
for row in rows:
|
||||||
|
if row["created"] is None:
|
||||||
|
del row["created"]
|
||||||
|
# A run row and another experiment must not participate in registration selection.
|
||||||
|
ignored = [dict(frozen, task=ex.RUN_TASK), dict(frozen, repo="experiments/other")]
|
||||||
|
registration_store.rows.side_effect = lambda *, repo, kind: [Record(r, "memory", None) for r in rows + ignored if r["repo"] == repo]
|
||||||
|
assert ex.registered(registration_store, manifest.id) is rows[expected]
|
||||||
|
|
||||||
|
|
||||||
def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None:
|
def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None:
|
||||||
store = FileStore(tmp_path / "memory")
|
store = FileStore(tmp_path / "memory")
|
||||||
store.put_rows(example_rows())
|
store.put_rows(example_rows())
|
||||||
|
|||||||
@@ -0,0 +1,291 @@
|
|||||||
|
"""Rule cache filling uses scripted clients and an isolated local store; no provider is contacted."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from learn_fixtures import ScriptedChat, default_answer
|
||||||
|
from test_learn_experiments import CODE, EXAMPLES, PRICES, excerpt_text, paid
|
||||||
|
from test_learn_rules import prepared, write_rule_manifest
|
||||||
|
|
||||||
|
from openultrasast.cli import main
|
||||||
|
from openultrasast.learn import experiments as ex
|
||||||
|
from openultrasast.learn.fill import fill_rule
|
||||||
|
from openultrasast.learn.program import Candidate, Program, Prompt
|
||||||
|
from openultrasast.learn.retrieve import Target
|
||||||
|
from openultrasast.plane.memory import FileStore
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot(root: Path) -> dict[str, bytes]:
|
||||||
|
return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()}
|
||||||
|
|
||||||
|
|
||||||
|
def seed_one(store: FileStore, manifest: ex.Manifest, unit: ex.Unit) -> int:
|
||||||
|
chat, caller = paid()
|
||||||
|
caller.store = store
|
||||||
|
example = next(e for e in EXAMPLES if e.id == unit.unit)
|
||||||
|
folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]}
|
||||||
|
program = Program(ex.arm_spec(manifest, manifest.arms["A"], unit.family, store), EXAMPLES, excerpt_text)
|
||||||
|
program.decide(Candidate(Target.of(example), CODE[example.excerpt_sha], "python"), folds[unit.fold], caller, seed=0)
|
||||||
|
return len(chat.calls)
|
||||||
|
|
||||||
|
|
||||||
|
def total(report: dict[str, Any], key: str) -> int:
|
||||||
|
return sum(f[key] for f in report["families"].values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_dry_run_makes_zero_calls_and_writes_nothing(tmp_path: Path) -> None:
|
||||||
|
store, manifest, units = prepared(tmp_path)
|
||||||
|
cached = seed_one(store, manifest, units[0])
|
||||||
|
chat = ScriptedChat()
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, dry_run=True)
|
||||||
|
assert report["status"] == "dry_run" and report["client_calls"] == 0 and report["usd"] == 0
|
||||||
|
assert not chat.calls and snapshot(tmp_path) == before
|
||||||
|
assert total(report, "units") == len(units)
|
||||||
|
assert total(report, "requests") == 2 * len(units)
|
||||||
|
assert total(report, "cached") == cached
|
||||||
|
assert total(report, "missing") == 2 * len(units) - cached
|
||||||
|
assert total(report, "filled") == 0
|
||||||
|
assert all(f["estimated_usd"] > 0 for f in report["families"].values() if f["missing"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_fill_only_missing_requests_then_replay_is_complete(tmp_path: Path) -> None:
|
||||||
|
store, manifest, units = prepared(tmp_path)
|
||||||
|
cached = seed_one(store, manifest, units[0])
|
||||||
|
chat = ScriptedChat()
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0)
|
||||||
|
assert report["status"] == "done" and total(report, "still_missing") == 0
|
||||||
|
assert total(report, "filled") == report["client_calls"] == len(chat.calls) == 2 * len(units) - cached
|
||||||
|
assert 0 < report["usd"] < 1.0
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
again = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0)
|
||||||
|
assert again["client_calls"] == 0 and total(again, "filled") == 0 and snapshot(tmp_path) == before
|
||||||
|
caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store)
|
||||||
|
replay = ex.run_rule(store, manifest, EXAMPLES, caller, meter, excerpt_text)
|
||||||
|
assert replay.decided == {"score": len(units)} and replay.skipped["replay_miss"] == 0
|
||||||
|
assert meter.calls == 0 and meter.usd == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("explicit_miss", [False, True])
|
||||||
|
def test_dry_run_uses_mean_billed_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, explicit_miss: bool) -> None:
|
||||||
|
store, manifest, units = prepared(tmp_path)
|
||||||
|
assert seed_one(store, manifest, units[0]) == 2
|
||||||
|
paths = sorted((tmp_path / "memory" / "responses").rglob("*"))
|
||||||
|
responses = [p for p in paths if p.is_file()]
|
||||||
|
assert len(responses) == 2
|
||||||
|
for i, path in enumerate(responses, 1):
|
||||||
|
entry = json.loads(path.read_bytes())
|
||||||
|
entry["usage"] = {"prompt_cache_hit_tokens": 9000 * i, "completion_tokens": 100 * i}
|
||||||
|
entry["usage"]["prompt_cache_miss_tokens" if explicit_miss else "prompt_tokens"] = (1000 if explicit_miss else 10000) * i
|
||||||
|
path.write_text(json.dumps(entry))
|
||||||
|
monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1_000_000))
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True)
|
||||||
|
family = report["families"][units[0].family]
|
||||||
|
mean_cost = 1.5 * (9000 * 0.014 + 1000 * 0.44 + 100 * 1.32) / 1_000_000
|
||||||
|
assert family["missing"] > 0 and family["estimate_method"] == "measured"
|
||||||
|
assert family["estimated_usd"] == pytest.approx(family["missing"] * mean_cost)
|
||||||
|
other = next(f for name, f in report["families"].items() if name != units[0].family)
|
||||||
|
assert other["estimate_method"] == "estimated"
|
||||||
|
assert report["client_calls"] == 0 and snapshot(tmp_path) == before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("usage", [None, {}])
|
||||||
|
def test_dry_run_falls_back_without_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, usage: dict[str, int] | None) -> None:
|
||||||
|
store, manifest, units = prepared(tmp_path)
|
||||||
|
assert seed_one(store, manifest, units[0]) == 2
|
||||||
|
for path in (tmp_path / "memory" / "responses").rglob("*"):
|
||||||
|
if path.is_file():
|
||||||
|
entry = json.loads(path.read_bytes())
|
||||||
|
entry.pop("usage")
|
||||||
|
if usage is not None:
|
||||||
|
entry["usage"] = usage
|
||||||
|
path.write_text(json.dumps(entry))
|
||||||
|
monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1000))
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True)
|
||||||
|
for family in report["families"].values():
|
||||||
|
assert family["estimate_method"] == "estimated"
|
||||||
|
assert family["estimated_output_tokens"] == 300
|
||||||
|
assert family["estimated_usd"] == pytest.approx(family["missing"] * (1000 * 0.44 + 300 * 1.32) / 1_000_000)
|
||||||
|
|
||||||
|
|
||||||
|
class FixedCostChat(ScriptedChat):
|
||||||
|
def complete(self, **kwargs: Any) -> Any:
|
||||||
|
response = super().complete(**kwargs)
|
||||||
|
self.usage[-1] = {"prompt_tokens": 0, "completion_tokens": 6_000}
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def test_hard_ceiling_stops_before_a_non_multiple_budget_is_exceeded(tmp_path: Path) -> None:
|
||||||
|
store, manifest, _ = prepared(tmp_path)
|
||||||
|
chat = FixedCostChat()
|
||||||
|
prices = {"cache_hit_per_m": 0.0, "input_per_m": 0.0, "output_per_m": 1.0}
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, prices, excerpt_text, client=chat, budget_usd=0.025)
|
||||||
|
assert report["status"] == "unfinished"
|
||||||
|
assert 0 < report["usd"] <= 0.025
|
||||||
|
assert report["usd"] == pytest.approx(len(chat.calls) * 0.006)
|
||||||
|
assert report["client_calls"] == total(report, "filled") == len(chat.calls)
|
||||||
|
assert total(report, "still_missing") > 0
|
||||||
|
assert total(report, "filled") + total(report, "still_missing") == total(report, "missing")
|
||||||
|
|
||||||
|
|
||||||
|
def test_smoke_failure_stops_before_any_fill_and_redacts_provider_error(tmp_path: Path) -> None:
|
||||||
|
store, manifest, _ = prepared(tmp_path)
|
||||||
|
|
||||||
|
def fail(*args: Any) -> str:
|
||||||
|
raise RuntimeError("provider unavailable SECRET_DO_NOT_PRINT")
|
||||||
|
|
||||||
|
chat = ScriptedChat(fail)
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0)
|
||||||
|
assert report["status"] == "failed" and report["client_calls"] == len(chat.calls) == 1
|
||||||
|
assert total(report, "filled") == 0 and total(report, "still_missing") == total(report, "missing")
|
||||||
|
assert snapshot(tmp_path) == before
|
||||||
|
assert "SECRET_DO_NOT_PRINT" not in json.dumps(report)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_units_refuse_before_calls_or_writes(tmp_path: Path) -> None:
|
||||||
|
store, original, _ = prepared(tmp_path)
|
||||||
|
path = write_rule_manifest(tmp_path, original.programs["injection"], "exp-902-pending")
|
||||||
|
manifest = ex.load_manifest(path)
|
||||||
|
ex.register(store, manifest, commit_of=lambda _: "c" * 40)
|
||||||
|
chat = ScriptedChat()
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
with pytest.raises(ex.ExperimentError, match="pending|frozen|units"):
|
||||||
|
fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0)
|
||||||
|
assert not chat.calls and snapshot(tmp_path) == before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("budget", [None, -1.0, float("nan"), float("inf")])
|
||||||
|
def test_paid_fill_requires_a_finite_nonnegative_budget(tmp_path: Path, budget: float | None) -> None:
|
||||||
|
store, manifest, _ = prepared(tmp_path)
|
||||||
|
chat = ScriptedChat()
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
with pytest.raises(ex.ExperimentError, match="budget-usd"):
|
||||||
|
fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=budget)
|
||||||
|
assert not chat.calls and snapshot(tmp_path) == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_zero_budget_stops_before_smoke(tmp_path: Path) -> None:
|
||||||
|
store, manifest, _ = prepared(tmp_path)
|
||||||
|
chat = ScriptedChat()
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=0.0)
|
||||||
|
assert report["status"] == "unfinished" and report["client_calls"] == 0 and report["usd"] == 0
|
||||||
|
assert not chat.calls and snapshot(tmp_path) == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_retry_requests_are_cached_for_identical_replay(tmp_path: Path) -> None:
|
||||||
|
store, manifest, units = prepared(tmp_path)
|
||||||
|
answers = 0
|
||||||
|
|
||||||
|
def first_invalid(messages: Any, temperature: Any) -> str:
|
||||||
|
nonlocal answers
|
||||||
|
answers += 1
|
||||||
|
return "invalid json" if answers == 1 else default_answer(messages, temperature)
|
||||||
|
|
||||||
|
chat = ScriptedChat(first_invalid)
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0)
|
||||||
|
assert report["status"] == "done" and total(report, "still_missing") == 0
|
||||||
|
assert len(chat.calls) == 2 * len(units) + 1
|
||||||
|
caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store)
|
||||||
|
replay = ex.run_rule(store, manifest, EXAMPLES, caller, meter, excerpt_text)
|
||||||
|
assert replay.decided == {"score": len(units)} and replay.skipped["replay_miss"] == 0
|
||||||
|
assert meter.calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_cli_dry_run_needs_no_endpoint_and_does_not_write_out(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||||
|
) -> None:
|
||||||
|
import openultrasast.learn.examples as examples
|
||||||
|
import openultrasast.model.endpoint as endpoint
|
||||||
|
|
||||||
|
_, manifest, _ = prepared(tmp_path)
|
||||||
|
real_get_blob = FileStore.get_blob
|
||||||
|
|
||||||
|
def get_blob(self: FileStore, kind: str, sha: str, **kwargs: Any) -> bytes | None:
|
||||||
|
if kind == "excerpts":
|
||||||
|
text = CODE.get(sha)
|
||||||
|
return text.encode() if text is not None else None
|
||||||
|
return real_get_blob(self, kind, sha, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(FileStore, "get_blob", get_blob)
|
||||||
|
monkeypatch.setattr(examples, "load_examples", lambda *_: EXAMPLES)
|
||||||
|
monkeypatch.setattr(endpoint, "resolve_chat_endpoint", lambda *_: pytest.fail("dry run resolved an endpoint"))
|
||||||
|
out = tmp_path / "report.json"
|
||||||
|
before = snapshot(tmp_path)
|
||||||
|
code = main(
|
||||||
|
["learn", "experiment", "fill", str(manifest.path), "--dry-run", "--memory", f"file://{tmp_path / 'memory'}", "--out", str(out)]
|
||||||
|
)
|
||||||
|
assert code == 0 and not out.exists() and snapshot(tmp_path) == before
|
||||||
|
report = json.loads(capsys.readouterr().out)
|
||||||
|
assert report["client_calls"] == 0 and report["status"] == "dry_run"
|
||||||
|
assert all(f["estimate_method"] == "estimated" for f in report["families"].values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_growing_call_costs_stay_inside_the_initial_reserve(tmp_path: Path) -> None:
|
||||||
|
store, manifest, _ = prepared(tmp_path)
|
||||||
|
|
||||||
|
class GrowingCostChat(ScriptedChat):
|
||||||
|
def complete(self, **kwargs: Any) -> Any:
|
||||||
|
response = super().complete(**kwargs)
|
||||||
|
self.usage[-1] = {"prompt_tokens": 0, "completion_tokens": min(len(self.calls), 9) * 1_000}
|
||||||
|
return response
|
||||||
|
|
||||||
|
chat = GrowingCostChat()
|
||||||
|
prices = {"cache_hit_per_m": 0.0, "input_per_m": 0.0, "output_per_m": 1.0}
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, prices, excerpt_text, client=chat, budget_usd=0.025)
|
||||||
|
assert report["status"] == "unfinished" and 0 < report["usd"] <= 0.025
|
||||||
|
assert report["usd"] == pytest.approx(sum(int(row["completion_tokens"]) for row in chat.usage) / 1_000_000)
|
||||||
|
assert total(report, "filled") == len(chat.calls) and total(report, "still_missing") > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_dry_run_prices_prompt_tokens_with_empty_cache(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.learn.fill import DEFAULT_OUTPUT_TOKENS
|
||||||
|
|
||||||
|
store, manifest, units = prepared(tmp_path)
|
||||||
|
report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True)
|
||||||
|
folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]}
|
||||||
|
index = {e.id: e for e in EXAMPLES}
|
||||||
|
expected = dict.fromkeys(manifest.families, 0.0)
|
||||||
|
for unit in units:
|
||||||
|
example = index[unit.unit]
|
||||||
|
program = Program(ex.arm_spec(manifest, manifest.arms["A"], unit.family, store), EXAMPLES, excerpt_text)
|
||||||
|
prompt, _ = program.prepare(Candidate(Target.of(example), CODE[example.excerpt_sha], "python"), folds[unit.fold], seed=0)
|
||||||
|
output_tokens = DEFAULT_OUTPUT_TOKENS
|
||||||
|
expected[unit.family] += (
|
||||||
|
program.spec.k * (prompt.estimated_tokens * PRICES["input_per_m"] + output_tokens * PRICES["output_per_m"]) / 1_000_000
|
||||||
|
)
|
||||||
|
for family, amount in expected.items():
|
||||||
|
assert report["families"][family]["estimated_usd"] == pytest.approx(amount)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bounded_provider_caps_output_and_disables_unmetered_retries(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.model.endpoint import DeepSeekChatClient
|
||||||
|
from openultrasast.provider.openrouter import OpenRouterChatClient
|
||||||
|
|
||||||
|
raw_calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
def raw(self: OpenRouterChatClient, **kwargs: Any) -> dict[str, object]:
|
||||||
|
raw_calls.append(kwargs)
|
||||||
|
assert self.max_attempts == 1
|
||||||
|
return {"choices": [{"message": {"role": "assistant", "content": ""}}], "usage": {"prompt_tokens": 20, "completion_tokens": 3}}
|
||||||
|
|
||||||
|
monkeypatch.setattr(OpenRouterChatClient, "complete_chat_raw", raw)
|
||||||
|
transport = OpenRouterChatClient(api_key="test-placeholder", max_attempts=3)
|
||||||
|
original = DeepSeekChatClient(transport, disable_thinking=True) # the default: config.models.thinking is False
|
||||||
|
bounded = original.bounded(300)
|
||||||
|
response = bounded.complete(model="deepseek-flash", messages=[{"role": "user", "content": "JSON"}], tools=[], json_object=True)
|
||||||
|
assert response.content == "" and len(raw_calls) == 1
|
||||||
|
assert raw_calls[0]["extra_body"]["max_tokens"] == 300
|
||||||
|
# The bounded copy keeps the configured client's thinking setting, so a fill answers as the run would.
|
||||||
|
assert raw_calls[0]["extra_body"]["thinking"] == {"type": "disabled"}
|
||||||
|
thinking_on = DeepSeekChatClient(transport, disable_thinking=False).bounded(300)
|
||||||
|
thinking_on.complete(model="deepseek-flash", messages=[{"role": "user", "content": "JSON"}], tools=[], json_object=True)
|
||||||
|
assert "thinking" not in raw_calls[-1]["extra_body"]
|
||||||
|
assert bounded.usage == [{"prompt_tokens": 20, "completion_tokens": 3}]
|
||||||
|
assert original.usage == [] and transport.max_attempts == 3
|
||||||
Reference in New Issue
Block a user