diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py b/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py new file mode 100644 index 0000000..cd9e857 --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py @@ -0,0 +1,53 @@ +"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking? + +Reads the run's arm_outcome rows from the memory store (OUSAST_MEMORY) and the frozen units file; makes no model call. +usage: python benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py (from the repository root) +""" + +from __future__ import annotations + +import collections +import json +from pathlib import Path + +from openultrasast.config import load_dotenv +from openultrasast.learn import experiments as ex +from openultrasast.plane.memory import open_store + +EXPERIMENT = "exp-004-pooled-block-gate" +UNITS = Path(f"plane/experiments/{EXPERIMENT}.units.jsonl") + + +def main() -> None: + load_dotenv(Path(".env")) + rows = [r.row for r in open_store(None).rows(repo=ex.experiment_repo(EXPERIMENT), kind="arm_outcome")] + with UNITS.open() as handle: + units = {u["unit"]: u for u in (json.loads(line) for line in handle)} + print("arm_outcome rows", len(rows), "units file", len(units)) + by_family: dict[str, list[tuple[float | None, int, str | None, str]]] = collections.defaultdict(list) + for row in rows: + unit = units.get(row.get("unit")) + if unit: + by_family[unit["family"]].append((row.get("s"), unit["label"], unit.get("pair"), str(row.get("verdict") or ""))) + for family, xs in sorted(by_family.items()): + scored = [x for x in xs if x[0] is not None] + pairs: dict[str, dict[int, float]] = collections.defaultdict(dict) + for score, label, pair, _ in scored: + if pair: + pairs[pair][label] = score + full = [p for p in pairs.values() if 0 in p and 1 in p] + wins = sum(1 for p in full if p[1] > p[0]) + ties = sum(1 for p in full if p[1] == p[0]) + top = sorted((x for x in scored if x[2]), key=lambda x: -(x[0] or 0.0)) + first_negative = next((i for i, x in enumerate(top) if x[1] == 0), len(top)) + print( + f"{family}: n={len(xs)} distinct_scores={len({round(x[0] or 0.0, 4) for x in scored})} " + f"unsure={sum(1 for x in xs if x[3] == 'unsure')} pairs={len(full)} within_pair_win={wins} tie={ties} " + f"pos_above_first_neg={first_negative}" + ) + pooled = sorted((x for xs in by_family.values() for x in xs if x[0] is not None and x[2]), key=lambda x: -(x[0] or 0.0)) + print("pooled raw top-30 labels:", "".join(str(x[1]) for x in pooled[:30])) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/record.json b/benchmarks/experiments/exp-004-pooled-block-gate/record.json new file mode 100644 index 0000000..8fcd18b --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/record.json @@ -0,0 +1,81 @@ +{ + "experiment": "exp-004-pooled-block-gate", + "date": "2026-10-03", + "fill": { + "client_calls": 2073, + "usd": 1.238223, + "ceiling_usd": 2.0, + "still_missing": 0, + "dry_run_estimate_usd": 5.582, + "note": "the dry-run estimator over-priced 4.5x (no prompt-cache pricing); measured spend is the meter's" + }, + "run": { + "units": 501, + "client_calls": 0, + "usd": 0.0, + "replayed": 2509, + "replay_miss": 0 + }, + "verdict": "inconclusive", + "reason": "no t_block on any outer fold, pooled or per family; B offers no BLOCK", + "instrument_check": { + "per_family": { + "access_control": { + "n": 66, + "distinct_scores": 41, + "unsure": 5, + "pairs": 28, + "within_pair_wins": 22, + "ties": 1, + "positives_above_first_negative": 12 + }, + "deserialization": { + "n": 64, + "distinct_scores": 38, + "unsure": 7, + "pairs": 25, + "within_pair_wins": 18, + "ties": 4, + "positives_above_first_negative": 5 + }, + "injection": { + "n": 149, + "distinct_scores": 52, + "unsure": 54, + "pairs": 53, + "within_pair_wins": 30, + "ties": 17, + "positives_above_first_negative": 6 + }, + "output_encoding": { + "n": 69, + "distinct_scores": 31, + "unsure": 32, + "pairs": 28, + "within_pair_wins": 18, + "ties": 6, + "positives_above_first_negative": 2 + }, + "path": { + "n": 66, + "distinct_scores": 43, + "unsure": 15, + "pairs": 29, + "within_pair_wins": 25, + "ties": 3, + "positives_above_first_negative": 13 + }, + "untrusted_destination": { + "n": 87, + "distinct_scores": 36, + "unsure": 33, + "pairs": 32, + "within_pair_wins": 18, + "ties": 4, + "positives_above_first_negative": 3 + } + }, + "pooled_raw_top30_labels": "111111111110111110111111110111", + "reading": "scores are real (31-52 distinct values per family); pooled raw top 30 holds 27 positives (0.90), first negative at rank 12; the Wilson bound needs >= 73 flags with no error, so precision at the top, not data volume alone, now binds" + } +} \ No newline at end of file diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/result.json b/benchmarks/experiments/exp-004-pooled-block-gate/result.json new file mode 100644 index 0000000..4d96898 --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/result.json @@ -0,0 +1,822 @@ +{ + "arms": { + "A": { + "label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p", + "rule": { + "name": "calibrated_block" + } + }, + "B": { + "label": "pooled gate -- one t_block over all families on a per-family Platt score (training folds, inner cross-fit), t_quiet per family at 90% recall, family floor 10 clean held-out flags", + "rule": { + "floor": 10, + "name": "pooled_block_gate", + "normalise": "platt", + "precision": 0.95, + "recall": 0.9, + "select_on": "paired" + } + } + }, + "code_commit": "90a2642019df414cb67b5dcea6e4ec235f35650a", + "decision": "inconclusive", + "experiment": "exp-004-pooled-block-gate", + "families": { + "access_control": { + "A": { + "calibration_holds": false, + "ece": 0.0715, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 66 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 66 + }, + "quiet_recall": { + "paired": 0.8929, + "pooled": 0.9211 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.44900641988143913, + "outer-1": 0.3723384028044276, + "outer-2": 0.4202772526614184, + "outer-3": 0.4481849709797749, + "outer-4": 0.36346174463707154 + } + }, + "capacity": { + "evaluated": 66, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 22, + "outer-1": 22, + "outer-2": 22, + "outer-3": 23, + "outer-4": 23 + } + } + }, + "deserialization": { + "A": { + "calibration_holds": false, + "ece": 0.0725, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 25, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 50 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 39, + "true_flagged": 0, + "units": 64 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 25, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 50 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 39, + "true_flagged": 0, + "units": 64 + }, + "quiet_recall": { + "paired": 0.88, + "pooled": 0.9231 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.17507194490500488, + "outer-1": 0.22766488322270723, + "outer-2": 0.2163745870743959, + "outer-3": 0.3112201878893598, + "outer-4": 0.363243722721776 + } + }, + "capacity": { + "evaluated": 64, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 20, + "outer-1": 20, + "outer-2": 20, + "outer-3": 20, + "outer-4": 20 + } + } + }, + "injection": { + "A": { + "calibration_holds": false, + "ece": 0.1042, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 41, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 53, + "true_flagged": 0, + "units": 106 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 78, + "true_flagged": 0, + "units": 149 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 41, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 53, + "true_flagged": 0, + "units": 106 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 78, + "true_flagged": 0, + "units": 149 + }, + "quiet_recall": { + "paired": 0.8679, + "pooled": 0.8974 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.287983504097999, + "outer-1": 0.17524866355153343, + "outer-2": 0.12918182665453967, + "outer-3": 0.14521040470628938, + "outer-4": 0.23005714687697673 + } + }, + "capacity": { + "evaluated": 149, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 38, + "outer-1": 44, + "outer-2": 45, + "outer-3": 43, + "outer-4": 42 + } + } + }, + "output_encoding": { + "A": { + "calibration_holds": false, + "ece": 0.0736, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 41, + "true_flagged": 0, + "units": 69 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 41, + "true_flagged": 0, + "units": 69 + }, + "quiet_recall": { + "paired": 0.8571, + "pooled": 0.9024 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.3723166587726901, + "outer-1": 0.5329072757249385, + "outer-2": 0.3785066578507705, + "outer-3": 0.4389011844848602, + "outer-4": 0.43855966643440264 + } + }, + "capacity": { + "evaluated": 69, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 23, + "outer-1": 22, + "outer-2": 22, + "outer-3": 22, + "outer-4": 23 + } + } + }, + "path": { + "A": { + "calibration_holds": false, + "ece": 0.0621, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 29, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 58 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 66 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 29, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 58 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 66 + }, + "quiet_recall": { + "paired": 0.8621, + "pooled": 0.8919 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.4680507207548809, + "outer-1": 0.3955717127313657, + "outer-2": 0.43694703358760895, + "outer-3": 0.4450583743716199, + "outer-4": 0.46396657833309796 + } + }, + "capacity": { + "evaluated": 66, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 23, + "outer-1": 23, + "outer-2": 24, + "outer-3": 23, + "outer-4": 23 + } + } + }, + "untrusted_destination": { + "A": { + "calibration_holds": false, + "ece": 0.1143, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 32, + "true_flagged": 0, + "units": 64 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 50, + "true_flagged": 0, + "units": 87 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 32, + "true_flagged": 0, + "units": 64 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 50, + "true_flagged": 0, + "units": 87 + }, + "quiet_recall": { + "paired": 1.0, + "pooled": 1.0 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.13676746287025585, + "outer-1": 0.11255775703980551, + "outer-2": 0.13763698150768486, + "outer-3": 0.09787938647374639, + "outer-4": 0.13763698150768486 + } + }, + "capacity": { + "evaluated": 87, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 26, + "outer-1": 26, + "outer-2": 24, + "outer-3": 26, + "outer-4": 26 + } + } + } + }, + "hypothesis": "One BLOCK threshold pooled across families on a per-family Platt-normalised score, chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95, reaches that bound on held-out repositories' paired units, where per-family thresholds (exp-003) could not for lack of flags; a per-family floor (>= 10 held-out flags, none wrong) keeps a family that the pooling masks from blocking.\n", + "manifest_digest": "87fceb6580c6de629d6a6089eb5888cdfc4df0f791576184f987c14c192d85f5", + "min_flagged_for_bound": 73, + "overall": { + "A": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 182, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 195, + "true_flagged": 0, + "units": 390 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 220, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 283, + "true_flagged": 0, + "units": 501 + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 182, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 195, + "true_flagged": 0, + "units": 390 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 220, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 283, + "true_flagged": 0, + "units": 501 + } + } + }, + "overall_after_floor": { + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 182, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 195, + "true_flagged": 0, + "units": 390 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 220, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 283, + "true_flagged": 0, + "units": 501 + } + } + }, + "populations": { + "primary": "paired", + "secondary": [ + "pooled" + ] + }, + "primary": [ + { + "name": "flagged_precision_lb", + "target": 0.95 + } + ], + "reason": "B offers no BLOCK on held-out paired units; the incumbent stays", + "resamples": 2000, + "secondary": [ + { + "name": "flagged_precision" + }, + { + "name": "coverage" + }, + { + "name": "flagged" + }, + { + "name": "quiet_recall" + } + ], + "spend_usd": { + "score": 0.0 + }, + "target_precision": 0.95, + "units": { + "by_label": { + "0": 218, + "1": 283 + }, + "evaluated": 501, + "excluded_replay_miss": 0, + "frozen": 501, + "groups": 220, + "not_run": 0, + "paired_evaluated": 390, + "unsure": 146 + } +} diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 2e9181f..335db09 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,7 +1,7 @@ { - "modules": 157, + "modules": 158, "counts": { - "load_bearing": 152, + "load_bearing": 153, "standalone_capability": 5 }, "orphaned": [], @@ -358,6 +358,7 @@ "learn.excerpt", "learn.experiments", "learn.features", + "learn.fill", "learn.folds", "learn.labels", "learn.leaks", @@ -412,7 +413,8 @@ "importers": [ "learn.cli", "learn.compile", - "learn.experiments" + "learn.experiments", + "learn.fill" ] }, { @@ -424,6 +426,7 @@ "learn.compile", "learn.evaluate", "learn.experiments", + "learn.fill", "learn.folds", "learn.program", "learn.retrieve" @@ -445,7 +448,8 @@ "classification": "load_bearing", "reason": "reachable from gate, map_gate, pair_gate, cli", "importers": [ - "learn.cli" + "learn.cli", + "learn.fill" ] }, { @@ -457,6 +461,14 @@ "plane.tasks.features" ] }, + { + "module": "learn.fill", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate, cli", + "importers": [ + "learn.cli" + ] + }, { "module": "learn.folds", "classification": "load_bearing", @@ -502,7 +514,8 @@ "learn.cli", "learn.compile", "learn.evaluate", - "learn.experiments" + "learn.experiments", + "learn.fill" ] }, { @@ -585,6 +598,7 @@ "findings", "learn.cli", "learn.features", + "learn.fill", "learn.labels", "model.candidates", "model.config_value", @@ -678,6 +692,7 @@ "importers": [ "cli", "learn.cli", + "learn.fill", "plane.budget", "plane.generate", "push.assistance", @@ -854,6 +869,7 @@ "learn.embeddings", "learn.examples", "learn.experiments", + "learn.fill", "learn.program", "plane.delivery", "plane.doctor", @@ -881,6 +897,7 @@ "learn.compile", "learn.embeddings", "learn.experiments", + "learn.fill", "learn.program", "plane.generate", "plane.tasks.roles", @@ -964,6 +981,7 @@ "learn.embeddings", "learn.examples", "learn.experiments", + "learn.fill", "learn.program", "plane.delivery", "plane.doctor", @@ -1819,6 +1837,8 @@ "reason": "reachable from gate, map_gate, pair_gate, cli", "importers": [ "cli", + "learn.cli", + "learn.fill", "model.endpoint", "plane.budget", "plane.tasks.roles", diff --git a/docs/decision-engine.md b/docs/decision-engine.md index 6186e35..fc2bdf8 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -150,7 +150,7 @@ Four experiments are registered: 0.875 against 0.906 and an interval spanning zero (`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`). - exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything. -- exp-004 tests one pooled BLOCK threshold with a per-family floor. It is registered and has not run. +- exp-004 tested one pooled BLOCK threshold with a per-family floor. It was **inconclusive**. ### exp-003: precision-bound blocking (2026-10-03) @@ -216,8 +216,44 @@ Calibration holds only for untrusted destination. There arm A still finds no BLOCK point that reaches the bound. Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json`. +### exp-004: pooled block gate (2026-10-03) + +The experiment tested one pooled BLOCK threshold across families. +Each family needed at least 10 clean flags to block. +Each family also had its own quiet threshold. +The run evaluated 501 units across 220 repository groups. +Of these, 390 units were paired. +The cache fill cost $1.238223 for 2,073 calls. +The replay cost $0.0 and made 0 client calls. + +The verdict was **inconclusive**. +No fold found a `t_block` threshold. +The instrument check confirmed real scores. +The pooled top 30 contained 27 positives. +The first negative appeared at rank 12. +The bound needs 73 clean flags without an error. + +A within-pair win means the vulnerable side scored above its fixed counterpart. + +| Family | Within-pair wins | Pairs | +| --- | --- | --- | +| access control | 22 | 28 | +| deserialization | 18 | 25 | +| injection | 30 | 53 | +| output encoding | 18 | 28 | +| path | 25 | 29 | +| untrusted destination | 18 | 32 | + +Precision at the top of the ranking now limits blocking. +Data volume alone cannot resolve this. +The next levers are graph evidence in the prompt and the combiner. + +Records: `benchmarks/experiments/exp-004-pooled-block-gate/record.json` and `benchmarks/experiments/exp-004-pooled-block-gate/result.json`. + ### exp-004: pooled block gate with a family floor (registered, not run) +This registration describes the setup before the run reported above. + The maintainer amended the BLOCK rule on 2026-10-03 (requirements, Req 6.4). Precision is now shown across all families together, not per family. Each family's score is first mapped by its own Platt fit, on training folds. @@ -230,8 +266,8 @@ No held-out label moves a threshold. - **Primary metric:** the pooled held-out Wilson lower bound of flagged precision, paired units, before the floor. - **Decision:** reject below point precision 0.95; adopt if the bound reaches 0.95 and a family passes the floor. -- **Units:** frozen at run time, from the memory after the negatives harvest. -- **Cost:** the run replays the cache. The new memory needs a paid cache fill first. +- **Units:** frozen from the memory after the negatives harvest. +- **Cost:** the run replayed the cache after a paid cache fill. Manifest: `plane/experiments/exp-004-pooled-block-gate.yaml`. @@ -252,6 +288,34 @@ The reading is on [Where we stand](where-we-stand.md#2-our-approach-to-detection | `ousast learn curve --program ID` | the metric over memory size on a fixed candidate subset | | `ousast learn audit-leaks` | how well each signal alone separates the sides of a pair (no model) | | `ousast learn experiment register\|units\|run\|analyse MANIFEST` | A/B experiments from a pre-registered manifest (`plane/experiments/.yaml`): record its digest, freeze the units (no model), run the arms paired (arm A may replay from the cache at $0), analyse (repository bootstrap, exact McNemar, two looks, the adoption verdict) | +| `ousast learn experiment fill MANIFEST --budget-usd X` | Fill missing responses for a registered `kind: rule` experiment's frozen units before its zero-cost run; `--dry-run` inspects and estimates without calls or writes | + +`experiment fill` accepts `--memory URL`, `--profile static|plane` (default `static`), +`--manifest-model PATH` (default `plane/models/deepseek-flash.yaml`) and `--out PATH`. +Freeze, commit and register the units first. Fill uses the incumbent programs and the +same retrieval, sampling and cache keys as replay. The first missing request is the +smoke call; a successful smoke response is kept. Any provider failure stops the fill. + +The JSON report on stdout gives each family's units, distinct requests, initially +cached and missing responses, newly filled responses and still-missing responses, +plus total client calls and metered USD. `--out` also saves it, except during dry-run, +which writes nothing. Exit codes are 0 for completion or dry-run, 1 for a provider/cache +failure, 2 for invalid setup and 3 for budget exhaustion. Resume a partial fill with a +new invocation and budget; that budget covers only the new invocation, including smoke. + +Dry-run uses each family's mean billed cached response cost when recorded usage exists. +Model prices account for cache-hit input, cache-miss input, and output tokens. +Without cached usage, it uses the program's input-token estimate and **300 output tokens**. +Each family's `estimate_method` reports `measured` or `estimated`. +It reports potential retries separately as `conditional_retries`: an unseen answer +may need one parse retry, whose necessity cannot be known without calling the model. +Known invalid cached answers include their retry requests in the inventory. + +Fill reserves at least **$0.01** and the largest observed call cost before each call. +For the production client it also reserves a conservative input-byte token bound plus +the program's enforced output-token limit; hidden provider retries are disabled. +Consequently a small budget may stop before making any call, and a fill may finish +with unused budget. Calls without token usage stop as failures, never as free calls. The memory store is the plane's (`OUSAST_MEMORY`; see [ops/ax/README.md](ops/ax/README.md)). The harvests that produced the plane signals ran as plane Runs generated by diff --git a/plane/experiments/exp-004-pooled-block-gate.units.jsonl b/plane/experiments/exp-004-pooled-block-gate.units.jsonl new file mode 100644 index 0000000..babb161 --- /dev/null +++ b/plane/experiments/exp-004-pooled-block-gate.units.jsonl @@ -0,0 +1,501 @@ +{"family": "untrusted_destination", "fold": "outer-3", "group": "adonisjs/http-server", "label": 0, "pair": "6b0b3cab2df3397b", "unit": "0ac37b7154242dfa637140af643fb4595e85510a17ab1a12c111441b9d293238"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "adonisjs/http-server", "label": 1, "pair": "6b0b3cab2df3397b", "unit": "6e5e274946d0e8f13a7ef4e37f6c7ed1828d057304a9262db0311413f18e1731"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "agentfront/frontmcp", "label": 0, "pair": "c0d0c91757f5ba95", "unit": "285891b16ee6870af50b36dec3c2ca9a90ee15d9e14851c61e66ad443e6eab59"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "agentfront/frontmcp", "label": 1, "pair": "c0d0c91757f5ba95", "unit": "f33fb2413edf1499a524c4592b3b2f93d166be845f5b65291969a086a4bec334"} +{"family": "path", "fold": "outer-2", "group": "ahujasid/blender-mcp", "label": 1, "pair": "f56f2a107ae591be", "unit": "350149bf9cafbb547f500b714a164734d2361b22e86fc9819c7bad3507d8458a"} +{"family": "path", "fold": "outer-2", "group": "ahujasid/blender-mcp", "label": 0, "pair": "f56f2a107ae591be", "unit": "ad125319dd2d4c9d2d99a1eb34bb4cfab9f099a96e7547d0d59a8ecf1c990347"} +{"family": "deserialization", "fold": "outer-3", "group": "aio-libs/aiohttp", "label": 1, "pair": "ee577ee4b6076fab", "unit": "2523f55d035d1a772a842090bc335e5d755b2fe15b376dd5b6b9e30b9dae82d1"} +{"family": "deserialization", "fold": "outer-3", "group": "aio-libs/aiohttp", "label": 0, "pair": "ee577ee4b6076fab", "unit": "ca0f6c160d7839204f6fcc1709bcbc749a02a0b3b8bb648e2a238eacbf22ed78"} +{"family": "access_control", "fold": "outer-4", "group": "alchemycms/alchemy_cms", "label": 1, "pair": "c95b6b01d0c2db2a", "unit": "15540c23be0c42472a49039658ec2ec0bbd8c6b318e35dd040a9d427370e0ef0"} +{"family": "access_control", "fold": "outer-4", "group": "alchemycms/alchemy_cms", "label": 0, "pair": "c95b6b01d0c2db2a", "unit": "542471980417e964182d7d3db59a05fe12bdd072449f3bd77f7daa5515dbc909"} +{"family": "output_encoding", "fold": "outer-0", "group": "angular/angular", "label": 0, "pair": "29252934ef4bcfcf", "unit": "63bae9240b31f9cf1dfe00d9b01d4d0ffda95931c9e7d040df1fd718f04d828b"} +{"family": "output_encoding", "fold": "outer-0", "group": "angular/angular", "label": 1, "pair": "29252934ef4bcfcf", "unit": "cce2af1743834dcc3df58827ac263a2d63296ca1b2a2f9bdb811954fb5767c33"} +{"family": "access_control", "fold": "outer-3", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "bcc636b23dfb5f570d8864b17fe07b6b4fb52b65a40988c32011f1422f74d504"} +{"family": "access_control", "fold": "outer-3", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "dbcf1e67a1be7ccf7d51701f9b8339339164055135443044c23a446b16b3d934"} +{"family": "injection", "fold": "outer-3", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "50408f68114ecb4de3bebd836d52d9afad86b22dd1843641ac8a49b617c09e33"} +{"family": "injection", "fold": "outer-3", "group": "anxolerd/dvpwa", "label": 1, "pair": "", "unit": "cafb0f00f80dc6beeba40eb95c764bf9a3569193070ba321c28677dfa425ec88"} +{"family": "deserialization", "fold": "outer-2", "group": "apache/airflow", "label": 0, "pair": "07317a2258292656", "unit": "a9f292d7cb467def582d9c0b31a397b13cdaefa77380ce971bfe9fbf535bb660"} +{"family": "deserialization", "fold": "outer-2", "group": "apache/airflow", "label": 1, "pair": "07317a2258292656", "unit": "ed363d0dfa620707be477b90eb4b83a901bfd402040b918aeee6f06b2c5b795c"} +{"family": "deserialization", "fold": "outer-3", "group": "apache/inlong", "label": 1, "pair": "1544b6dfa2299d60", "unit": "d45018fc4ce3dfd715c7087d4d197ad4108c0c8b6625c02c4d4da0adb9f6a140"} +{"family": "deserialization", "fold": "outer-3", "group": "apache/inlong", "label": 0, "pair": "1544b6dfa2299d60", "unit": "f295c7dfb095595a4aa985c58688e4757c29be9e06b7a4ca55035ac5d06b1ef8"} +{"family": "output_encoding", "fold": "outer-4", "group": "apache/logging-log4net", "label": 1, "pair": "fe0e104d111c2555", "unit": "088036fab9962be2b96f9162ea4f32190575678d3c5ba96bdfeac178bc22b2c4"} +{"family": "output_encoding", "fold": "outer-4", "group": "apache/logging-log4net", "label": 0, "pair": "fe0e104d111c2555", "unit": "7fd2b56a816e25358477eee64487fb2c77f34c652703ad0a7120c368492017a3"} +{"family": "path", "fold": "outer-2", "group": "apache/wicket", "label": 1, "pair": "b6742570f7f0e026", "unit": "7c6c1d27aee6557f1b6bf371b8209003f0d2d1122cfa35b6629eb43f38769cf4"} +{"family": "path", "fold": "outer-2", "group": "apache/wicket", "label": 0, "pair": "b6742570f7f0e026", "unit": "df8300d84dd784d0be46969ea432ef97b8ce611bae7a896ed4f349bd4d6897db"} +{"family": "access_control", "fold": "outer-4", "group": "apolloconfig/apollo", "label": 1, "pair": "0f584bb97a154488", "unit": "222a84ffc7c58d45e8f10df221762b23bf20d0287ecd20f375558134928886ca"} +{"family": "access_control", "fold": "outer-4", "group": "apolloconfig/apollo", "label": 0, "pair": "0f584bb97a154488", "unit": "25e4c9d2941f59f072385a7bde3fd0bd76f9e986ed24b06ce81962ab447f5094"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "appium/java-client", "label": 0, "pair": "b15b924b4f7b1ae1", "unit": "2fa95fce0751c97db1a12a80ddb249ff5c54ee31c7dc3cd47765099c7b673e52"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "appium/java-client", "label": 1, "pair": "b15b924b4f7b1ae1", "unit": "52be8128053102eafbee67c2d3aac88c83ca290b943ab2f2cfdcd2630601c7d9"} +{"family": "access_control", "fold": "outer-3", "group": "arcadedata/arcadedb", "label": 0, "pair": "d31ca58f2a1d27d1", "unit": "2ae40ea61d691060e7b9f6c8e910e3aa0edf16bc6f88bec81aa9a7dae5cf5603"} +{"family": "access_control", "fold": "outer-3", "group": "arcadedata/arcadedb", "label": 1, "pair": "d31ca58f2a1d27d1", "unit": "4876728701698045e65eeae6e11e133c30f43b09029d7777d6d780f7d0855885"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "archivy/archivy", "label": 0, "pair": "bb3d7baed62e6bb7", "unit": "e8ae88fd2b677e3f64ac94bddb012e3b903afa294ecc87774d10087c0d50489c"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "archivy/archivy", "label": 1, "pair": "bb3d7baed62e6bb7", "unit": "f6b0d241891dc1d897d74d98b6ff7ada2c1f1ce7874ca32097630c06ca53e389"} +{"family": "injection", "fold": "outer-1", "group": "argos-ci/argos-javascript", "label": 0, "pair": "d9d8cfa38981847f", "unit": "864f1800617f5218a2300ce390ba7922fe1bd0a182de115952342b61d627bb16"} +{"family": "injection", "fold": "outer-1", "group": "argos-ci/argos-javascript", "label": 1, "pair": "d9d8cfa38981847f", "unit": "fdcda2307e20ef1b9247b7082a85337f457dd1f4e648bb8264077bf983c03271"} +{"family": "access_control", "fold": "outer-2", "group": "arikusi/deepseek-mcp-server", "label": 0, "pair": "1f73ad2576d52a64", "unit": "0bc9634940734cf7f41628f220032fb7e2fd2ff5b584afc519875e90ea0be5a4"} +{"family": "access_control", "fold": "outer-2", "group": "arikusi/deepseek-mcp-server", "label": 1, "pair": "1f73ad2576d52a64", "unit": "ddb59b2dc39715281a6d48b32e7ca4ec0354b85b953fff55139579f83905b7d6"} +{"family": "access_control", "fold": "outer-0", "group": "arthurkatcher/google-maps-mcp", "label": 1, "pair": "a7326e6daa507b79", "unit": "793ebf4c8e0f46d8ce3b3d090d2f6290d4167931a3b574f8269b90008e3dcde7"} +{"family": "access_control", "fold": "outer-0", "group": "arthurkatcher/google-maps-mcp", "label": 0, "pair": "a7326e6daa507b79", "unit": "b8d25f337fad527544c33c82f473015e08399457276fc480477f3cfcfc9c5b8a"} +{"family": "path", "fold": "outer-1", "group": "asymmetric-effort/nogginlessdom", "label": 0, "pair": "211ba072bdfa67b7", "unit": "3217f9d672a4a1d78005c8fb66e1bc60a542cb26bd8c46903402c765c123d60a"} +{"family": "path", "fold": "outer-1", "group": "asymmetric-effort/nogginlessdom", "label": 1, "pair": "211ba072bdfa67b7", "unit": "8370f9624baa0d144e56b8087eea897b98e5b2afcd6eb3a029d03e5b4b69d584"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "authlib/authlib", "label": 1, "pair": "baccf350d5cc9911", "unit": "50a3ca70700c8b41874c43ba94e6b7ffe0aa28b8930ff941a40767e1057065b5"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "authlib/authlib", "label": 0, "pair": "baccf350d5cc9911", "unit": "62927a118d3662c81450e19d49f4146b66f29a58d55ec0d1606ed521b1883f12"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "axios/axios", "label": 1, "pair": "697f34e19bda57f3", "unit": "775b6d900ec1ba71e4082ed49bb94439f38066628eed91461eecf0658caee78a"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "axios/axios", "label": 0, "pair": "697f34e19bda57f3", "unit": "9e855586c685b3c62ffe32407a82966661d5839e2cff742f2676acaba645a97a"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "axllent/mailpit", "label": 1, "pair": "143b900b99a46cf4", "unit": "200f96cd784ff5c39d8bba22883f22bc8a7c87c6970a7c871802f48a45548337"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "axllent/mailpit", "label": 0, "pair": "143b900b99a46cf4", "unit": "7a0d62ec424849ec4681f4333fbb077e8239054d1e964bd99b40847ad7ccdb01"} +{"family": "output_encoding", "fold": "outer-3", "group": "basecamp/trix", "label": 1, "pair": "4b4e81797ca48cbf", "unit": "a4d17d85973d024a615b89334ad5a2f46afea660ef5d0bcf2b2196c998b1caf2"} +{"family": "output_encoding", "fold": "outer-3", "group": "basecamp/trix", "label": 0, "pair": "4b4e81797ca48cbf", "unit": "d604e460d8f5291124ba6fab5f9c94c7dfda61ea5f6808e512162334d6d6db4f"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "beaugunderson/ip-address", "label": 1, "pair": "bcbd06e53147628b", "unit": "0b5cfe2d64771e30e50ff92cc409b8dd5e30079c252918ba03774cb5198a73b5"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "beaugunderson/ip-address", "label": 0, "pair": "bcbd06e53147628b", "unit": "b805976c35330452af6c9f814ff4617b7b9ad0e82e1fa4440a443f93f0772c4a"} +{"family": "injection", "fold": "outer-3", "group": "browserify/static-eval", "label": 1, "pair": "c12dbc8a50f7fe9b", "unit": "04beb8a0fd054b92c2dd0ed83907330fdee6124d86114df948ac72a297a7d51f"} +{"family": "injection", "fold": "outer-3", "group": "browserify/static-eval", "label": 0, "pair": "c12dbc8a50f7fe9b", "unit": "c5029016a6fd2e3f9df59dba0ce3c173bbbb0d2b1de22d151086e01e1b272119"} +{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "0e213aeabacf4205db024ea3c557e720c3aba911424555784917e967d6404867"} +{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 0, "pair": "e8bc65415bd42e43", "unit": "4920230ef773595a9fd9a92a090dff42b48499a1dc5fc8831f941dae55cb2538"} +{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 1, "pair": "e8bc65415bd42e43", "unit": "49209a810f207eb61382e2ceeab7076936838a5652546f21c187d066e220160b"} +{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "9e2964884cdf281fa3f18ee3481aa62f9a8a0a195d605da2df5bd71876fe311a"} +{"family": "injection", "fold": "outer-4", "group": "budibase/budibase", "label": 1, "pair": "", "unit": "f9cc6f1bbcd59f5dc0de8f96248f722ea0679f8afa47c53f70832151ce7616d3"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 0, "pair": "e69463f945f9a0f4", "unit": "5feace7b28e260175f80c682e9009025aea6fae3dc75409cabbb955b560d0372"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 1, "pair": "e69463f945f9a0f4", "unit": "799a4f5a5b9324e86c90752f0239f059984f44e8ff1ec0dd518d0d9413e34c66"} +{"family": "injection", "fold": "outer-3", "group": "cakephp/cakephp", "label": 0, "pair": "71f56810b8ee6947", "unit": "c659c0e717b7dc1084e1178288f6c9b2e87fbf0ef518ba3bbaedb0fbadae15c1"} +{"family": "injection", "fold": "outer-3", "group": "cakephp/cakephp", "label": 1, "pair": "71f56810b8ee6947", "unit": "c8d2e035a6099fb775c8b4ecf00196c1488dbddd9eed3d2d268a4f1d2567b194"} +{"family": "deserialization", "fold": "outer-1", "group": "cesargb/laravel-magiclink", "label": 0, "pair": "768fc9db8a75a351", "unit": "adc6527469618f72b1dd7776814717cbd0056a0c625e05bef74199171ce73f1a"} +{"family": "deserialization", "fold": "outer-1", "group": "cesargb/laravel-magiclink", "label": 1, "pair": "768fc9db8a75a351", "unit": "da47663a57e370a75cc65b810b8a380aa26d12dcabb1d9b3c02641398ff7d348"} +{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 0, "pair": "fab4ce020049724b", "unit": "b9be90242be627aa46462143ac85940a906232ebc0b47776b5729475b27d215e"} +{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 1, "pair": "fab4ce020049724b", "unit": "ece12ea7d87b4a12c4da30faed9eb530b51fc879c271b68ba5304b230cda8e79"} +{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 1, "pair": "9126ba83905eb8b4", "unit": "d8dc50b7bb40bbcfc3f3d4f1f8d3a4d17eb1c662d5f759efd18656a482dd2e32"} +{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 0, "pair": "9126ba83905eb8b4", "unit": "f426f9307621b3f2d6a8d1669cdb4c19e962b94e34fbb394d4660cbde5921c88"} +{"family": "deserialization", "fold": "outer-1", "group": "commenthol/serialize-to-js", "label": 0, "pair": "7ac5e21d9d8f4c47", "unit": "1d1efa2fd7323c264290d4d314c957c853e1c7a9c877a54d6f03ab7d2af2ca86"} +{"family": "deserialization", "fold": "outer-1", "group": "commenthol/serialize-to-js", "label": 1, "pair": "7ac5e21d9d8f4c47", "unit": "b5dc78c7ac0b76803dc98904142c1139083fb9daafb5bff40e7b5c3f14ee03f5"} +{"family": "deserialization", "fold": "outer-1", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "0894a7212280bab755cd9d93b1f0ede2c51e4fca76b76e5ed9e24ef8f8cf0b9c"} +{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "02694b5fb85e9e52bf5eaaab4e0b628f1f44ff3a8d5ad026cc10afdbc4222eaf"} +{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "db9a3e888780501dd3c2c125af2f9d47f68ebf02f0e0db8370b2ea09c6f9d070"} +{"family": "injection", "fold": "outer-2", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9af96b099dedcc4005e5fcd19ad78f73864e1cfbafbe6953b4edb6260957943d"} +{"family": "injection", "fold": "outer-2", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9e7c2a786068a936a6c664cc229d6f39cea1fe57caf0806306f0cfc44515460c"} +{"family": "deserialization", "fold": "outer-3", "group": "cps-it/mailqueue", "label": 1, "pair": "07d1dc4d299a7700", "unit": "0e664e274dfb438d2f721edc81f8469ac48387dbd5c1b43973cd7e651db46db3"} +{"family": "deserialization", "fold": "outer-3", "group": "cps-it/mailqueue", "label": 0, "pair": "07d1dc4d299a7700", "unit": "da55f60378c67e91683b9720f2fa0e6659e28c7b392c050278b40713e47c84a3"} +{"family": "output_encoding", "fold": "outer-3", "group": "cstigler/node-xhtml-purifier", "label": 1, "pair": "749eee05a507fe93", "unit": "24fbb1dffcca083382a23e48db232e0d83bbbf2ff2b325f0dc0f4c72dd1a16bc"} +{"family": "output_encoding", "fold": "outer-3", "group": "cstigler/node-xhtml-purifier", "label": 0, "pair": "749eee05a507fe93", "unit": "640e1626f8d7d9d5c2478ede48cc7b2780370bda98ae114bae102bba892da90c"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "czlonkowski/n8n-mcp", "label": 0, "pair": "e36bbaf8f3cfe65c", "unit": "14f7c65fb5621ac5e2068c8695ddbf482103e4320ec3b36886f6e1f08cbc312d"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "czlonkowski/n8n-mcp", "label": 1, "pair": "e36bbaf8f3cfe65c", "unit": "c96dd379dec2ecfd99051d7a20e6ef5601821f93c46eb34ed087daf5fbed2d47"} +{"family": "injection", "fold": "outer-4", "group": "danger/danger-js", "label": 0, "pair": "b81c8c63cae892f6", "unit": "5982fe7fa1ea6bfc1c8dcb8c71cd4410de475d25cafa324147649f26a1ae1324"} +{"family": "injection", "fold": "outer-4", "group": "danger/danger-js", "label": 1, "pair": "b81c8c63cae892f6", "unit": "59916deeb8ba14922eebac61c9eac122e15d1f7590ee3cb1b97f019585fb798f"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "danielgatis/rembg", "label": 1, "pair": "bab3a167ab5ca505", "unit": "5ebea23aee96c80fa0d2d9239964d99614613c4ae36f709811bc19e61ebc720c"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "danielgatis/rembg", "label": 0, "pair": "bab3a167ab5ca505", "unit": "9bfea8d7a6ffa6d79d16f77fc355fc1bb605189513ebad561ee523d577a1498b"} +{"family": "deserialization", "fold": "outer-3", "group": "datapizza-labs/datapizza-ai", "label": 1, "pair": "5a91049be21aa611", "unit": "1ab9936627aa59c3a60ea44074ad44b68aea4f9c04f14c1949dd9c77ca13ff0a"} +{"family": "deserialization", "fold": "outer-3", "group": "datapizza-labs/datapizza-ai", "label": 0, "pair": "5a91049be21aa611", "unit": "8df72b9eab3f2fd858c40312d0a806cb67c620bd62873826c8732ba8795d9cfa"} +{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 0, "pair": "", "unit": "115d87f37bf1d4ab91fa3010303e81b127f84b817268b60b8fd4b9934efeae4f"} +{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 0, "pair": "2b8101c89c839614", "unit": "7bb7d858a670fc51f81e08c141c2bca3b8b7524254ab679e33b22ce79536e895"} +{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 0, "pair": "c23acf4f38945093", "unit": "82bdc0b6bb0c19e342efa63af0c9014a607461d0682cdd14bd5fea33516fbd50"} +{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 1, "pair": "2b8101c89c839614", "unit": "886285661b88f14e00ff8e863b31cf7b7afb303ad4aa4aa13b1cffbf1272d5e0"} +{"family": "injection", "fold": "outer-3", "group": "davila7/claude-code-templates", "label": 1, "pair": "c23acf4f38945093", "unit": "bf74aae5a48156f183426a80cafca6da219ab13b2ae4b40fa88fca75acb72980"} +{"family": "access_control", "fold": "outer-1", "group": "delmaredigital/payload-puck", "label": 0, "pair": "d7cc37a3a54315cb", "unit": "4aecd43c6e5b81daf7aadd6a6e22f22c17c08b652a2f5121a49320ca1057b186"} +{"family": "access_control", "fold": "outer-1", "group": "delmaredigital/payload-puck", "label": 1, "pair": "d7cc37a3a54315cb", "unit": "88123ae860be2fdc28fd6a289cf26c3e0af0689f455cf8c15098e5dc76e5c8a8"} +{"family": "path", "fold": "outer-4", "group": "dep0we/atomic-agents-stack", "label": 1, "pair": "5011004f06f1e217", "unit": "6c48682d1b02ff2559d92346d192cc9dbe6a0bbe2ab2c146653110420105ff1d"} +{"family": "path", "fold": "outer-4", "group": "dep0we/atomic-agents-stack", "label": 0, "pair": "5011004f06f1e217", "unit": "d3592260108a838246e0cfd5ab6d48bceb0ef870e223565745dcfd68f1826840"} +{"family": "output_encoding", "fold": "outer-1", "group": "devbridge/jquery-autocomplete", "label": 0, "pair": "81caad96b221c2b8", "unit": "3c2dc720ed159e55629ba737665d93859def9790427289567a513c83c133e236"} +{"family": "output_encoding", "fold": "outer-1", "group": "devbridge/jquery-autocomplete", "label": 1, "pair": "81caad96b221c2b8", "unit": "e8b9595a320776c8e663fd0b168193c1e64e36a9052784d1219f2c4a91d647f2"} +{"family": "path", "fold": "outer-4", "group": "dgtlmoon/changedetection.io", "label": 0, "pair": "f9b6599018ef3b60", "unit": "7558d6ee39a311c5459e35fe6f7b12917b7f843b21ab1fbb5e485a07f53b7d78"} +{"family": "path", "fold": "outer-4", "group": "dgtlmoon/changedetection.io", "label": 1, "pair": "f9b6599018ef3b60", "unit": "b1a6cf450a074788ffca2bebd880435abac2c376e89cb328d7a22d277fdccad8"} +{"family": "output_encoding", "fold": "outer-4", "group": "dicebear/dicebear", "label": 1, "pair": "eaf2f03db2c502b5", "unit": "38dbfbf745d7cbcc52c6fff83248d84663273fe6a1f1b014eac7776c4bb8f074"} +{"family": "output_encoding", "fold": "outer-4", "group": "dicebear/dicebear", "label": 0, "pair": "eaf2f03db2c502b5", "unit": "a84d848ef56b72fc15872d88acaae1743c9cdf88095d13b1869abbec6a0d8a8f"} +{"family": "deserialization", "fold": "outer-0", "group": "docling-project/docling-core", "label": 0, "pair": "7df48bca2c9fabfc", "unit": "7b76ce8f55d61071adb367018cc3e50f04a05d08c7c4919c3806fc1cc8bf41a7"} +{"family": "deserialization", "fold": "outer-0", "group": "docling-project/docling-core", "label": 1, "pair": "7df48bca2c9fabfc", "unit": "c82ea0adc9c8366aa2b77ce73772038676fc37ff50879e873ff397958fdd7ec1"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "7f02a09b8d8f2321f2cdade74c2fa1d52442e006149a749a8be768daaa62195d"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "9b5e439943ce5c2b04d6201b7a843634aa1b13b726813c23450ff54ce03f3dd6"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "5b836889b4da4921", "unit": "a84c2ce4a4eadf265d4481ad7f6864edf1d7e3084c6ee32cbbfe454bb2946bfe"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 0, "pair": "5b836889b4da4921", "unit": "caeed7f097a3311fe2e84efd372fe4b4c140f63a80a9f07ba0b61966dbab15e0"} +{"family": "injection", "fold": "outer-4", "group": "electerm/electerm", "label": 0, "pair": "ac526f457067fad8", "unit": "50d8bafd5189fab0429bd4135d9ed400cc2fc14706226a2de98d65746ebfdcc7"} +{"family": "injection", "fold": "outer-4", "group": "electerm/electerm", "label": 1, "pair": "ac526f457067fad8", "unit": "9532b70b0df76a34170b57c74deafaba54d5d2f16a729c39b0dfd3eb2022d899"} +{"family": "path", "fold": "outer-1", "group": "elyin/gemini-bridge", "label": 1, "pair": "c97a971a18f85244", "unit": "95c80c503ed43fe9a0370f89ea935d47badee6a9625548b9dca78e04275003dd"} +{"family": "path", "fold": "outer-1", "group": "elyin/gemini-bridge", "label": 0, "pair": "c97a971a18f85244", "unit": "e9810872beb9c158f54048392142c2f701314f854d6dfc748553ca36ed6c08d0"} +{"family": "access_control", "fold": "outer-4", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "c2a2c2d85114a97f892fb1e5500fbd98622457a8c46451e1f9e958ab08a40318"} +{"family": "injection", "fold": "outer-4", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "29d8be8206c88ba7efd86b56a7dcf8b655e33b4a792c2d18b028c92c6252daab"} +{"family": "access_control", "fold": "outer-2", "group": "esphome/device-builder", "label": 0, "pair": "6a4e0bd705331747", "unit": "6db80439d89b54f60f836c415dd403fd09d21cf630da81435d4d2f9cef5d883c"} +{"family": "access_control", "fold": "outer-2", "group": "esphome/device-builder", "label": 1, "pair": "6a4e0bd705331747", "unit": "bf257e4f0812012be330fe119d07d29253d291161f435262787f491762c6bfb1"} +{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 0, "pair": "f96f0fb245948915", "unit": "0954e6931373915c204b9ec8d2b8ae6297a95667bfd41aadf9035625ad93967f"} +{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 1, "pair": "f96f0fb245948915", "unit": "5e09cdae2504e834d591b6f92a808427f07550364b5419899f7e6f76084fc377"} +{"family": "output_encoding", "fold": "outer-0", "group": "ether/etherpad", "label": 0, "pair": "bf11f138fe39675e", "unit": "56f0d0b156992af56141d6368fdb0c5376fc7a44a03991d754f97202b9d77008"} +{"family": "output_encoding", "fold": "outer-0", "group": "ether/etherpad", "label": 1, "pair": "bf11f138fe39675e", "unit": "8271af5017894167556893da2070188060d06eebc87f27db45c46234c359631d"} +{"family": "path", "fold": "outer-0", "group": "fastify/fastify-static", "label": 0, "pair": "9d937c2dffb4aadc", "unit": "01ac12bb06e660d0aeef8b1e26dddc5c99d52cc333f7af6d1981eee2818e64fd"} +{"family": "path", "fold": "outer-0", "group": "fastify/fastify-static", "label": 1, "pair": "9d937c2dffb4aadc", "unit": "9612edf0f927048335e83e9925f01be9aea9b772bdbdc568ad9cc0a17055662a"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 1, "pair": "d4f648048e45f2ea", "unit": "08807c8110b8c1b8635a6883c229fb91f2b9566db8b6c252448a0db7b2ddd47e"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 0, "pair": "d4f648048e45f2ea", "unit": "10869c931053bf77ecaa98a47f56dea27d643d1abed2a4758bcf48a78547ccc9"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "662e1f36c230d7f1", "unit": "0b73332b25413902c26d5b5dee6767465eed99ff71a814582988ffe0d8ec1a30"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "8c8e46d0c7d361b784baff9d70f38744b2805ef1ee0aaacaa616bc20a2c6f32d"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "aca938682c196fd4c85b39b89eeedb97c35f6a5a017651fe9c03c5b727cab7ea"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "cec00d1509a129364da6ab8537fb3f433fd847e26d073c90311ccdaf7a1b39e7"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 1, "pair": "662e1f36c230d7f1", "unit": "e0f58a7ac5627de3e7b6cfe5add41d18b0f64b998c1cc788d42eb2cdbbc3fe41"} +{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 0, "pair": "2c251a645094f389", "unit": "0d41cc634771048f72d07c4aa9feaed669d0bfe3844d61dcdc525a0f899617e2"} +{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 1, "pair": "2c251a645094f389", "unit": "ca8288d0b8b94dc6b7e4c681936ea610d295fe949cba5f64f09b6de134ef9875"} +{"family": "access_control", "fold": "outer-4", "group": "forgekeep/nebula-mesh", "label": 1, "pair": "2ac5ec5011615c9b", "unit": "2f427f403af3cb58d776437736c2389805543413fdb91aad98979158c6d00de3"} +{"family": "access_control", "fold": "outer-4", "group": "forgekeep/nebula-mesh", "label": 0, "pair": "2ac5ec5011615c9b", "unit": "cddc8c5bed0841b80e7b20d0b378e858f8e4ac37fd7b890ce76611d989857f92"} +{"family": "injection", "fold": "outer-3", "group": "frangoteam/fuxa", "label": 1, "pair": "", "unit": "0c7830c2912ff285855d8a3c8e9b47db4ca4af840baa66d4b5d3f4422f46f95d"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "5383a8c2a78fabd3d17a5b3021125953b5ebbee61f8900c30d016c56b1a3bad6"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "360d8bfd2f116e98", "unit": "7c1989a4ec3622bdaebcf1161de850f22cfed041cb38830ca1ded4fa99d40466"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "ec8c706cc2b20f36", "unit": "848688dcd0fe8d46e6762cc8aee1d82e8fad8a2a044c94b547aea11049c85bfc"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "ec8c706cc2b20f36", "unit": "983cc9231f91baee4e2d626193f3b88aa5b09907f6faab870949bdc9c336b816"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "e4ad86416d5c6668", "unit": "98ff1dff2afe80709b8e61a9c95cfeea12b2110c13b7120478268b4762e2c992"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "360d8bfd2f116e98", "unit": "a1a825d23e34757c8e961e43ae791253e62682e130acdb846b58118588006a98"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "a7fa4207508b3989baabc7a1f0ff2ea5229b366c1cb106e7fc945134b8b106c8"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "e4ad86416d5c6668", "unit": "ef5c53f3114d0057e694df13dc07cbcb532c3b0d59150eda269b87b021d7a6e8"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "getgrav/grav", "label": 1, "pair": "9ae575048504c690", "unit": "176acf077fd58c3e6f940753179ffa3e8316661f355d94acb5e0345b51925ca4"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "getgrav/grav", "label": 0, "pair": "9ae575048504c690", "unit": "228b34ebc2090b5239c7821d28366ddc663e7118d015a79b45dca6618d5490fb"} +{"family": "access_control", "fold": "outer-1", "group": "getgrav/grav-plugin-api", "label": 0, "pair": "d6edb5496b3fdba2", "unit": "9acb8017ebadc480515a664181c427f37359edcc1dc6f344f368a87e0c191b3f"} +{"family": "access_control", "fold": "outer-1", "group": "getgrav/grav-plugin-api", "label": 1, "pair": "d6edb5496b3fdba2", "unit": "c1bfe63148bc700c812fde95ad9f83d456e4fa04e10c6f5394072a08f73174c6"} +{"family": "injection", "fold": "outer-0", "group": "gitpython-developers/gitpython", "label": 0, "pair": "27e1c8e8ee93a892", "unit": "47fb89b132d6de48674c37965579e727d922bef1796db70f8b9f862ac47cb9a5"} +{"family": "injection", "fold": "outer-0", "group": "gitpython-developers/gitpython", "label": 1, "pair": "27e1c8e8ee93a892", "unit": "79887f0ca8b05d6599d55167d8a9faf67bdb2254d17c143141da44971077002d"} +{"family": "output_encoding", "fold": "outer-3", "group": "gofiber/template", "label": 0, "pair": "a3fdf9ce3441ec86", "unit": "653236018929ab8534c27704a20ec5d358047bc66e451620d2ac32081c91d333"} +{"family": "output_encoding", "fold": "outer-3", "group": "gofiber/template", "label": 1, "pair": "a3fdf9ce3441ec86", "unit": "b70ca2c43f5929e7b885bd728349e721fd85d7605e2f722a7e8cfe0f0d477b0c"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "groundhoggwp/groundhogg", "label": 1, "pair": "", "unit": "92400e83890882a66ffdea1a405e2354a195442e176d85fb707b0442c44d45ca"} +{"family": "path", "fold": "outer-4", "group": "gtsteffaniak/filebrowser", "label": 0, "pair": "f9dcf3a0fb0f0a14", "unit": "7ca6dd4f673976b000a657f885af3463740419ce41e14f972af0c5483cb75a7d"} +{"family": "path", "fold": "outer-4", "group": "gtsteffaniak/filebrowser", "label": 1, "pair": "f9dcf3a0fb0f0a14", "unit": "86c1db3a5e9f843d6546146449171e0378ac61d6c8f44c6dd72fd379cdbb188a"} +{"family": "deserialization", "fold": "outer-4", "group": "h2oai/h2o-3", "label": 1, "pair": "3e71f79348a6b770", "unit": "200250ebceadde4cbdfec2802a4f9fbfa220dbd7fa4c2dd5ed717fb1f3ea7447"} +{"family": "deserialization", "fold": "outer-4", "group": "h2oai/h2o-3", "label": 0, "pair": "3e71f79348a6b770", "unit": "b9faa5007c8cbb67ada3f1f026ca59e0ace6fbb9c3d1f1369d4521a7e323e7da"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "h3js/h3", "label": 0, "pair": "8972a0f56ee57477", "unit": "6b987df2960828dcb1b2b3f190a903a40361d58117d63321d671aecc6c2cb570"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "h3js/h3", "label": 1, "pair": "8972a0f56ee57477", "unit": "ca3fa87dd37435c01a7121119912c1a1e261259c11805c0b66e7d4b315640945"} +{"family": "injection", "fold": "outer-1", "group": "handsontable/formula-parser", "label": 0, "pair": "6144baeb491f4a56", "unit": "f5c22ce104e9593e5272b6e8b3c6f3b61fa4578552ccea80e6021ee90bdb3ffc"} +{"family": "injection", "fold": "outer-1", "group": "handsontable/formula-parser", "label": 1, "pair": "6144baeb491f4a56", "unit": "f687e7e95dbe47f602a1e0002a44f94fd48e5b1c33f77ecebc6c2d814fcbf450"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "heartcombo/devise", "label": 0, "pair": "2ef4471c66e5e4ab", "unit": "ad7c8e368fe1699a044e058d5c10b41bd9fc0f91c3f3c384c5dac40af39e71f6"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "heartcombo/devise", "label": 1, "pair": "2ef4471c66e5e4ab", "unit": "d888baf7194bbe7158cb3932f58b47a9a235351d388d10df82a130ddd54c2d46"} +{"family": "output_encoding", "fold": "outer-2", "group": "honojs/hono", "label": 1, "pair": "5628c54152216646", "unit": "47d081c9f6060218967902f694e61c88f64f57c65bec6fc3073afeb0b77d117c"} +{"family": "output_encoding", "fold": "outer-2", "group": "honojs/hono", "label": 0, "pair": "5628c54152216646", "unit": "e1297bccf9c3606e0b41d762a69ff5d0a33ae4f7f1506071af6b9cb4a177ff56"} +{"family": "path", "fold": "outer-4", "group": "hs-web/hsweb-framework", "label": 1, "pair": "e52aa814c36ace1f", "unit": "2d6d74364f1982e91f27f1c995b8699df1fc8f8dce6e1e2cf274bf01b6e9494e"} +{"family": "path", "fold": "outer-4", "group": "hs-web/hsweb-framework", "label": 0, "pair": "e52aa814c36ace1f", "unit": "4727471fea2cf0a92d43b4b1503f1339a7dde87f872ba8064dacbf855c57705b"} +{"family": "path", "fold": "outer-2", "group": "huggingface/datasets", "label": 1, "pair": "a07929b3fc1cfc28", "unit": "0637029194bd7378237b4dc944dc5cf9f2b63db9796f27410a74a9d567b66dc4"} +{"family": "path", "fold": "outer-2", "group": "huggingface/datasets", "label": 0, "pair": "a07929b3fc1cfc28", "unit": "ffd286c8588a77929639a89e4a742e15306b642af14aa906c721165df804ccd3"} +{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 0, "pair": "6548e1527e82bd99", "unit": "5542613f6d2eb9421b8213b8bb864894f0ecf9fe64945681019230ef0d75cfaa"} +{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 0, "pair": "", "unit": "6c2adee34e9e8384535ec1073eda60e2ab528ac77c36747802c303e1f5e9ffe8"} +{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "7ce70e9b20107cc7ebb87d173da765fc53ffd268e521749a06a9a1f38d746845"} +{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 1, "pair": "6548e1527e82bd99", "unit": "c3e54975464323e58ec96ee4d83e7bc08baf94553d553a8dc3fb28cd9c70ff21"} +{"family": "injection", "fold": "outer-2", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "d3173c1e43eb6402a71ea2dec1592913dc078a2194ba1c1d509ab129a3180f6f"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "internlm/lmdeploy", "label": 1, "pair": "0924befc49c7bb18", "unit": "e7e4872db40f0420547ed622fbdd7a072b64a7ba466d1ccfff68e691cee6ad99"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "internlm/lmdeploy", "label": 0, "pair": "0924befc49c7bb18", "unit": "f7d95f43fb2df20ebc4c37fceabdd00ffd8159f404ca384481f1764252382388"} +{"family": "injection", "fold": "outer-3", "group": "iximiuz/node-diskusage-ng", "label": 0, "pair": "d4ee6cfeba71e398", "unit": "e79582de3031bada2617a0667ef7ff5cd7183d4aa885f55a665cf0ec88d17624"} +{"family": "injection", "fold": "outer-3", "group": "iximiuz/node-diskusage-ng", "label": 1, "pair": "d4ee6cfeba71e398", "unit": "fef35272853e7f3692efd54daa01c12862953352fa6ccbbb5e3f7fea70c63d78"} +{"family": "path", "fold": "outer-0", "group": "jarofghosts/glance", "label": 1, "pair": "772e18821701d60f", "unit": "89a0496c7663bbfd67d660c9f481c12420da2dd1bdb29dc079735ee7a0a49626"} +{"family": "path", "fold": "outer-0", "group": "jarofghosts/glance", "label": 0, "pair": "772e18821701d60f", "unit": "d3241ceaa50a7d23a4650e6a2a55372b08ec96b35b0bede844bc40e16e818ebc"} +{"family": "injection", "fold": "outer-1", "group": "jas-/node-libnmap", "label": 1, "pair": "250de1302c7e541d", "unit": "5d03535f2df2acdbc319d9e6b20da65d51f3e810b80f3c8f89b5074a971a3f1c"} +{"family": "injection", "fold": "outer-1", "group": "jas-/node-libnmap", "label": 0, "pair": "250de1302c7e541d", "unit": "78f977cfc096e34b4eada7f798ad2973ef45728d0d01bfab6ac9e2a3257827ef"} +{"family": "injection", "fold": "outer-2", "group": "jashkenas/underscore", "label": 1, "pair": "c10bfb006f2da55a", "unit": "1ea4b86dbd84581b1d7b7cabbab9f28c8b213416a60778879666ff42d23fd7f1"} +{"family": "injection", "fold": "outer-2", "group": "jashkenas/underscore", "label": 0, "pair": "c10bfb006f2da55a", "unit": "7647021f742da24c660b2d407886771786e957076c4cf336def560a570dd8778"} +{"family": "access_control", "fold": "outer-2", "group": "jellyfin/jellyfin", "label": 1, "pair": "a73421efb106ab0a", "unit": "1f6f207a758582de0c0e6b331b19fe119abe18727d45ccbbe09aabb795f3a292"} +{"family": "access_control", "fold": "outer-2", "group": "jellyfin/jellyfin", "label": 0, "pair": "a73421efb106ab0a", "unit": "32fbdad39eaf5177db4ad0a11d53e785b30bc7b2376ad9d460f29a08429bd0c7"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/cas-plugin", "label": 0, "pair": "0458e3fd444409f9", "unit": "b521216bb4106738b3571ad3902d08f6230aabd88cf3a8f9f26f1c8ec01304cb"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/cas-plugin", "label": 1, "pair": "0458e3fd444409f9", "unit": "c87d2871907798fa2e73d31381e9d63089b0388033f096d247373fec07c2cb36"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/google-login-plugin", "label": 0, "pair": "6bdccabcf5b9fdc9", "unit": "3d6ddca5eb264d88411b86755909f1d2abddd09c129cabd54c96ff26b4105316"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "jenkinsci/google-login-plugin", "label": 1, "pair": "6bdccabcf5b9fdc9", "unit": "e055b898802279bc63204cc601e29f7f1018efad7228cab0cce00ea5866ecc7b"} +{"family": "access_control", "fold": "outer-3", "group": "jenkinsci/mcp-server-plugin", "label": 1, "pair": "91bc832e70680c92", "unit": "1060c63ff0fae3af567448f3d4c60d9c159b61ccc358964243eb152a9ac38865"} +{"family": "access_control", "fold": "outer-3", "group": "jenkinsci/mcp-server-plugin", "label": 0, "pair": "91bc832e70680c92", "unit": "369dffd1e730d7997813adfbebf1481cc68987cae0d0085e47b91a26d5330cf8"} +{"family": "output_encoding", "fold": "outer-0", "group": "jihong88/suneditor", "label": 0, "pair": "137fb3496a3a75f4", "unit": "7e8b6c484999229b23d50d9f5a4c4c5b2a8d954a5d64230981b1c6705b760fbf"} +{"family": "output_encoding", "fold": "outer-0", "group": "jihong88/suneditor", "label": 1, "pair": "137fb3496a3a75f4", "unit": "c461154e4dbd74e35a4ed2e85de7b076f0582beeec459be28104c38f616c7565"} +{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 1, "pair": "a4144258dffb3b8d", "unit": "7ab301e1614979f44c353e665540582493f78ef4bfc41209562c19b900710f8c"} +{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 0, "pair": "a4144258dffb3b8d", "unit": "ea95e8387dff32ded9dc7b04c11a76a545b4e2ea6db0076b9b13971a59904bb3"} +{"family": "deserialization", "fold": "outer-4", "group": "jmespath/jmespath.rb", "label": 1, "pair": "858330e01d3cf9bb", "unit": "033cda2b7cffa5a5fa6abf0b7f2cd27ba78d916eb3cab8c79decccd8776f4a3f"} +{"family": "deserialization", "fold": "outer-4", "group": "jmespath/jmespath.rb", "label": 0, "pair": "858330e01d3cf9bb", "unit": "d44c4555a5ae19aa17b468ec5cff56b33a0b221df52e04d8abcb656399499dee"} +{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 1, "pair": "c6a49339d4f0c798", "unit": "dc2e4cd5bd963643702d2b832ae170dd25744162e4076e36687256a09e69d089"} +{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 0, "pair": "c6a49339d4f0c798", "unit": "f83cd137b85e651ca5299c5a42176b797ed4964cbef31418c96d63c52fdb65f5"} +{"family": "access_control", "fold": "outer-1", "group": "jowilf/starlette-admin", "label": 1, "pair": "4b6526d132e9ac86", "unit": "8ec3596444f3045684242840e9a1df19879383f2733aca8f8af6f6cf8a96d388"} +{"family": "access_control", "fold": "outer-1", "group": "jowilf/starlette-admin", "label": 0, "pair": "4b6526d132e9ac86", "unit": "f3d39162fa14446b207aabe7bf3c913dc32a9637ac83352ea5cb2fdbb43720a8"} +{"family": "injection", "fold": "outer-4", "group": "jsonata-js/jsonata", "label": 0, "pair": "80abf1d5c4a3f6a5", "unit": "3df3c6d9941352dd2861427da56b94ee335f308d0a2a27c6f745aef60349c80f"} +{"family": "injection", "fold": "outer-4", "group": "jsonata-js/jsonata", "label": 1, "pair": "80abf1d5c4a3f6a5", "unit": "4fde61a570906e658d9b14db0e5d1f5acc0c7489951421bff2afde25033ff73c"} +{"family": "access_control", "fold": "outer-2", "group": "jsonbored/gittensory", "label": 1, "pair": "5ec37c0f475001ca", "unit": "5d8175fb21a4649bdc8730358e9b87adb90b2aafc4da5fee203829663e4f46a0"} +{"family": "access_control", "fold": "outer-2", "group": "jsonbored/gittensory", "label": 0, "pair": "5ec37c0f475001ca", "unit": "af845e9e332555096dd350401977b2f07015433f3db8a040891d9470f3f9e355"} +{"family": "deserialization", "fold": "outer-0", "group": "keras-team/keras", "label": 1, "pair": "68d44ef338ef3470", "unit": "3020ac6c3186d7974953b1c4da65b80141b81aca5203436ae7a2d7aca0870944"} +{"family": "deserialization", "fold": "outer-0", "group": "keras-team/keras", "label": 0, "pair": "68d44ef338ef3470", "unit": "8fdb85a6bfec89f0715ea81e83fd91616f92c59954c092e79272154f202b7488"} +{"family": "deserialization", "fold": "outer-4", "group": "kevin1024/vcrpy", "label": 1, "pair": "c74525e844650de2", "unit": "2e444ad34ee5f47d73f0cfad0a10c70363337b4ddf9067ecd1b8072d8232652c"} +{"family": "deserialization", "fold": "outer-4", "group": "kevin1024/vcrpy", "label": 0, "pair": "c74525e844650de2", "unit": "c492d325f5ff82b5f817aa4be1440cec8595054dd0719e07a58ac5501b2ae7eb"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "koajs/koa", "label": 1, "pair": "111a5a80a3abc2c9", "unit": "c7ae403742450deb2bd051ad1231121d2117ba56fa72a038c82d58d207cb3870"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "koajs/koa", "label": 0, "pair": "111a5a80a3abc2c9", "unit": "debe34831afca6f8220b9952036a2d4570babdb5e321a2f8dbaec661a016bd50"} +{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "9950b85412b021ebabb11eea4c490a660ad380d14c673a87c3fb568ba4bcfac9"} +{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "656df5ff8e69dcdb1f9ba1ed52d734966eb29773bc745587483a355743b8eb5a"} +{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "91652a4e776fbce70cde9e4a99c6da5e17da7f8d6d0404867c61eec04161b6cf"} +{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e53bb3a217dc4737e8dd03671de4f6a895599024070a985a268c65a034295e7c"} +{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "3d7aba22444811a337abcad0e4fc62990bb704d3a10e72d893342fa94699026b"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "5f18a3026a51f7140b7aaa3aa8563e8b6fdda6065d95e19e25b1dac4b9ce0e00"} +{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "0fccc306ce80cb4cbfdcde0a2e56d637bcefa9dd30423d6c54a4f77860afe740"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "1e2eaa45661345f1b6759d968d7696dd0d96d6fba1350a00e4d0a2ebaaa2e391"} +{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "8b9a8ff0e32a327df75eb8a1811464d5f34690079d5930139dd917063bdd2809"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "245d0e939b7c1ccf098c798492009e3a73ea17400fa613dd865cd369d2e7f383"} +{"family": "access_control", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "653c48df493f31c2d01c973d1e379a180fbe6a6edc83a500bc62509112175f41"} +{"family": "path", "fold": "outer-2", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "3719fe21d45b50b35458bcd2e263e4baee67adbd3c98adf0eaab6bc6183c5651"} +{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "1209a6cc8c1548ed83d4a9a09d6a27f706a23e69cf0c261fc5e3a27536a83306"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "362c90495dc468bb732c71f62d71ec308854161de31ad18d2a870056d947ddb6"} +{"family": "access_control", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "b52e53fec9707b2b442c4dc7a41d376cf76946d2c45c2eb600c9477fc6baebaa"} +{"family": "path", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "477e3b90b92e622888dbe0e9d3a23b31271ddfa3104a5ab326eb9e9e765aa01d"} +{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "660c63c4a73bfe64088ae3249e40d9c5d69a6f8a505ff671217c5708a345c5b0"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "4d60c386c801bedb61c1461fced29a3cfb50b09d6435e8617c60398343719d93"} +{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "f1fddcc1354a29bef55610ee2b73eaa440192f2f18a0d22d08057b9d1e1b61b8"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "6d9886c38221b2f9aacf83749218ac5ebc354a6fcc4170e6c78bedb92b88296a"} +{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "5550fec4a81c4c9cb22d7312d8a63e1fb2b03c4aed0c1f316d44c1eb27511bac"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8ff87dd04140f0137ea1891f220b1276156ce10a58be5a49d0c8e07be531ffb2"} +{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "f24b19cc0f67247eccb274521a62741fef8f4729f0ddc9b5c1fa561a4947182a"} +{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "83523c9d83380d4cdae593a593d361493e00d18e61740f84e762aad41b991e18"} +{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "c6c9bde6630678377d250701073c5c223c7892e6ff8f2f8e0293dea22c5b5e7e"} +{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "b9b15958973c83f8d7b58372ab5ba58eb191c400125579d5ae8c5b52d5d1368e"} +{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "b95ec12e4e126ad558e944e2c260116ed6a63f5f3ab97fb5cbcdca75a931ad73"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "38a5545ac9ef67e15db720d1396a2d5915f9d06d6f4772012e5bc95f33770376"} +{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "c8bbe9be91c976be97b7ca387048a7d4537d9f6fcc6cbad0a541d1080629d5a6"} +{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "ffacd53f1bd75155e63b318fb9cba27ec03e754841e1eb39d1d4fa0b2c3dfca2"} +{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8fd920175fc43b91b4397acffacb10ab8f7fd0ee38bcf2d2f3ed06a3fad0dfe9"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "86c9598ebe3dcda261a69ae545513f6ce73101f3980066e4506ece99bebea84f"} +{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e5190fcc57505116f79a31c51aed69f8a1eea13a8ae3e7265f44c72418277ef9"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "2b807ff54e6c47e941124c16d0c27c691cef54f2203060a3a6ce812d478c5261"} +{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "401aefb4b4791301ace2d770d321c6b26743b0aab7b96f8b410254c704be6d2e"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "22d186c93498e21418b6c1d0547a73374b13041d8afc89a80d9540fa658b6b42"} +{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "dbfb6df14fe37f8c0efda5f16643297fb0550dfc5ad5805d4e4b844d967773b2"} +{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "f0da8cfe4ac1e586aa7903d72e67b44500bbd5306d72b4b342d10613efc69222"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "2b8fd73943f456da266434470333a7775f55d9a41953e97a000fdaa3257b6a30"} +{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "a612091f452e60f28f44eefec641175170dc5db04456a39a099cbb13c50b4d2c"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "4c6600facd83bbf234a32d9e28ac494fb516df6e3d9e9dc28d720ff35252d067"} +{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "fb0ea8cab8c9430d22d1b0fdd65a69146fa95fbff94e720b4861b96538d992c6"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "e81f91a56058ac1473579767abea57064c53d7d007797a854314814e3c35ee74"} +{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "5851ecd96a449f98b05f243bfda0a7cf3af8612b27f53c414ae1f4ea67242a7f"} +{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "328621ef2a94632e5d37ea6267d17b766233384cde35c07e29a4eb672d40ba28"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "04b2eae2128608dad460ba8275142a41a44e32c4243a29e2a6fb2c9ebf3d8f1a"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "33337002ff5f5a189610b042522ceea254c9d107c1f24771b4f616a0f96f636e"} +{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "02c09a783cbf058856fc2b448406f3ffa6fbfbacf9b70f4e5e22fd741a5c7699"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "956260164cb21a47c499e5598491031ef303418943f5096f225a87162b588574"} +{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "8bd8c5aed69eae3019cc2d3654b36bdf2b31435e6a020a389cd103696a2f874c"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "1d8397ef4a9fde7b48e2e1e871bba0f960a802cd7823ecfc10e9dfaddb955ccc"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "c9db92476032eb4296b950a1eff8fa4aaa26f11542a01a5cf10c7b2e32dc6433"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "d3e0d0f9b0f4b723080989b5486395cda0a5b186a674022ea70e7fa7ab561323"} +{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "44ff3b3e61c02744830c4abea872821ca7e0f94950c082740c5c48ef13604e94"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "4103e979a99233c3e71fdfe9f1cdcbab34a2c60b5351b75c5ff5dfc287b962d0"} +{"family": "injection", "fold": "outer-0", "group": "koxudaxi/datamodel-code-generator", "label": 0, "pair": "f73ecbfda2bcadef", "unit": "53e596dabf90b2992d5b3e50394791859b393beedc8f84e00484e37498627ff0"} +{"family": "injection", "fold": "outer-0", "group": "koxudaxi/datamodel-code-generator", "label": 1, "pair": "f73ecbfda2bcadef", "unit": "6daaccf64706e876dd505c65a62aeacc1357f3470a5cef7b1ac874032a6fcf99"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kozea/weasyprint", "label": 1, "pair": "6768bfae38c6859d", "unit": "f376519889a00950f5af67eb15c639306ea70da7235d3be4a94dbc19cb096cf5"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kozea/weasyprint", "label": 0, "pair": "6768bfae38c6859d", "unit": "f3fea4e334f49ef6fe6b17121fef553eb8e33a9ffa13e6d6cbc4c448da996a7c"} +{"family": "access_control", "fold": "outer-0", "group": "kubev2v/migration-planner", "label": 1, "pair": "d670253fa209671c", "unit": "1aeebe2e440bdd70525c31f93180f33be93f820db5ba46a2e59c371ef83ee794"} +{"family": "access_control", "fold": "outer-0", "group": "kubev2v/migration-planner", "label": 0, "pair": "d670253fa209671c", "unit": "b11c8a8a24d3809c0507228568fe7cc05ca0a22ed28cb19019adb13cb619d63f"} +{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 0, "pair": "33260f3e6c8ea6d0", "unit": "7ca409c7dbc1c38ff005624e212efc6d424cb421e27b482dc8dc0d6881f4c11a"} +{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 1, "pair": "33260f3e6c8ea6d0", "unit": "fc364d7ed77c7c0508da00e5ba847ec0792f3bc373d8b4297747189ef42d1ae0"} +{"family": "access_control", "fold": "outer-1", "group": "labs64/netlicensing-mcp", "label": 0, "pair": "32ef8148a2c08f58", "unit": "2fb2c9314a483787168a450f50430721f9527f913aa0eae07298d5334cf4f137"} +{"family": "access_control", "fold": "outer-1", "group": "labs64/netlicensing-mcp", "label": 1, "pair": "32ef8148a2c08f58", "unit": "78dd291bb2ef83ea3a87948c0ac0027c6d667b9044093e766e2bc91bebe83936"} +{"family": "path", "fold": "outer-3", "group": "langchain-ai/langchain-nvidia", "label": 0, "pair": "54586b5a24655adf", "unit": "2ece75a0240ed36a811c381841aa6d440af2b9774ddacb1d7a31a67ec267b0b2"} +{"family": "path", "fold": "outer-3", "group": "langchain-ai/langchain-nvidia", "label": 1, "pair": "54586b5a24655adf", "unit": "e60f6b788a196db9bb671d621189b79a5526302a6d09c2396d67fbe0b9ab3c24"} +{"family": "access_control", "fold": "outer-2", "group": "langflow-ai/langflow", "label": 0, "pair": "474f96e2a9b3d202", "unit": "4a5e8e8ebd7ecb0e252280af0cb38a1d7794757728b912525322ac26df5ee223"} +{"family": "access_control", "fold": "outer-2", "group": "langflow-ai/langflow", "label": 1, "pair": "474f96e2a9b3d202", "unit": "6a888540443061d3b37c06f0cebc70cffdfa70e0aa940626e86cbff9829310fe"} +{"family": "deserialization", "fold": "outer-1", "group": "laravel/reverb", "label": 1, "pair": "a09692dc94164db8", "unit": "139cf84e79c4efd026a064b2ca0a3d7ffff87b54eae1275a07c0d69ef4981e27"} +{"family": "deserialization", "fold": "outer-1", "group": "laravel/reverb", "label": 0, "pair": "a09692dc94164db8", "unit": "828e88b656245a1d0865c79b2ed023a49e78607a4ac81d70ccd3690f4419d2d1"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "", "unit": "6cce78d7569ad8db7ca0a61d7078198f666502d26ea6c0afb03173a8a4bb16ef"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "82047c0513e43a83", "unit": "6d2dbed1ab1c9d8ac259dd4227be69641fa59d52e816a8dc547ade01d0b9b0c4"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "98ee23cd6bd31cd3", "unit": "ab4bb39be535d70cd95e9035bb958bfc7418576a6ff40a47551894dd140317bb"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "98ee23cd6bd31cd3", "unit": "b70068cbc646bd01d3e10986e002f648a74e9d798900dc59cf3039b749c95c18"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "ff168d95d8d9496e", "unit": "c6ec9478c125ed780d7013c2807f2cec1878f44253667dfed319c28cbfd17d4d"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "b1bfef46dcd3b9a5", "unit": "cd4d095d8380a2fb7ca02fc6abdbae5249cc06455db2929875533f4aa8b886dc"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "82047c0513e43a83", "unit": "d1bade217be8d345cf5d0f0442ee31924187acbb98779dada95274a77462f696"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "ff168d95d8d9496e", "unit": "d561db250b892b91feed7ae877900e1e25963444f73c7699576e010f06da73cf"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "b1bfef46dcd3b9a5", "unit": "ed78fd5a7ab30bcd6e8ddd27c706a19cfd89750de4fbf81c0b567946eb94db83"} +{"family": "output_encoding", "fold": "outer-0", "group": "lepture/mistune", "label": 0, "pair": "25f75d77a669b54e", "unit": "0f2825568461562f2a915a4f16ef8a4931290fd20a9801470aeb12677e66e3f1"} +{"family": "output_encoding", "fold": "outer-0", "group": "lepture/mistune", "label": 1, "pair": "25f75d77a669b54e", "unit": "bf22bb44bcff4a5baf5d550c9d3e83e5f3ca607475fb51deaf4fb6fccff3d73a"} +{"family": "path", "fold": "outer-1", "group": "linuxfabrik/lib", "label": 0, "pair": "47069df6bfb3dc00", "unit": "444c1f41fc23ad656a8bf98dae58b36d0f1daa214ab030682f26111dd389e58d"} +{"family": "path", "fold": "outer-1", "group": "linuxfabrik/lib", "label": 1, "pair": "47069df6bfb3dc00", "unit": "c0c8b5cca070673ea4b4bb8f08dfff86b7781780e300720c4051aded035052fe"} +{"family": "output_encoding", "fold": "outer-2", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "cd80750447cf7af67ad809279718d98106c71705fef9ff7254728bfff4ece5b1"} +{"family": "path", "fold": "outer-2", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "2ce605ab5ce7ad3a1fb983cd6bddf3924e5da3010d2252a0a4640aab4e5921a0"} +{"family": "deserialization", "fold": "outer-0", "group": "litedb-org/litedb", "label": 1, "pair": "b0a899116dc36b85", "unit": "510ad48af03ba4a1a2a06dd254b135311dddb62d9fe5a2e8e8b12627586e6b13"} +{"family": "deserialization", "fold": "outer-0", "group": "litedb-org/litedb", "label": 0, "pair": "b0a899116dc36b85", "unit": "b414718a8a5f79739b88db24da6ad9da8e48bcb6b5e2dc99158848cdaa226767"} +{"family": "deserialization", "fold": "outer-0", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "8e22ee30441b20484090659b0c97423c711f5d10219041bf3a399623fc056a07"} +{"family": "injection", "fold": "outer-0", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "0f9070df9dc334f981cb01008c260b8bff78dbdeb48ce672830ed95caa8406fb"} +{"family": "injection", "fold": "outer-0", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "5e352f797b1422b0df477bce4730a9ce33cbd0837c3d63da94e4d8801a9a4b64"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 1, "pair": "dadccb128ac0e120", "unit": "505097494da5939404ad7d5b4af2b1c4a10efe94048084492d6ce60ebaa04c97"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "dadccb128ac0e120", "unit": "84ff8ca2d98a1b5d1ca4bfaabe7d19e03a396042b228c8d18d1185a178389702"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "91e261c54bd2c3c3a9ac04e48894ecabaca5b13e7db4fbd5578f9d266098117a"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "a308dfd59bc398824e2454bf6cf6cfd5b7880e5b948a82424f14753d2b6fcdbe"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "ba24eb554458bc97314ed561a7b2f4a4e643dde1205c32de4c8fcdd8c46e575b"} +{"family": "output_encoding", "fold": "outer-1", "group": "mganss/htmlsanitizer", "label": 0, "pair": "24d8f1840b67fbe5", "unit": "7ffa4a229a62266788fbf64890f81f20358b3eb8c35d3ab87d0c339d5bb4b507"} +{"family": "output_encoding", "fold": "outer-1", "group": "mganss/htmlsanitizer", "label": 1, "pair": "24d8f1840b67fbe5", "unit": "d19493ed7941e1cd25a76dc50504d97398770d07ebc302dd66d83441e946c468"} +{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 0, "pair": "9f8a60eb48d74bc8", "unit": "80fce34f2377731e3de4355c7ea33bb20fc2fc00a5f388565b23018c7a3ed008"} +{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 1, "pair": "9f8a60eb48d74bc8", "unit": "b68e62722a327c9d5ce6774eb96527451729410768efc0b5c7ba604ec468052d"} +{"family": "injection", "fold": "outer-4", "group": "microsoft/prompty", "label": 0, "pair": "2c857235d0764986", "unit": "33bdfe911cd1b512e5994ee7a3e3ab2ad1f302cab29df3b011c0d825c23536a3"} +{"family": "injection", "fold": "outer-4", "group": "microsoft/prompty", "label": 1, "pair": "2c857235d0764986", "unit": "b5efdcb84a057f3cd9fd3a30a32239dc06c13536a4e84a66c7a17cb372050c97"} +{"family": "output_encoding", "fold": "outer-0", "group": "middleapi/orpc", "label": 1, "pair": "9e84429d629c0b6b", "unit": "774cfdecac67499bb35a872931d4698d1b92ebeeb42d21790469e418889954e5"} +{"family": "output_encoding", "fold": "outer-0", "group": "middleapi/orpc", "label": 0, "pair": "9e84429d629c0b6b", "unit": "a6c5c15ca1815f83dfb47428c4a8afd4e60fa1c027b9f33c668cb9a0d77c18af"} +{"family": "output_encoding", "fold": "outer-2", "group": "mixxorz/slippers", "label": 0, "pair": "8e6b4b4e28981590", "unit": "3cce3e8f9ff3f1d261f9b96c844c1e4da07bae72d5393897b3a929be25372edc"} +{"family": "output_encoding", "fold": "outer-2", "group": "mixxorz/slippers", "label": 1, "pair": "8e6b4b4e28981590", "unit": "b105689ac24ed334294c434a77bb6d56d5d598830ffe8a5d60ab3be5290b47ad"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "0a36cd5177f3b2c20b433db6bfffe5ec5b7e74cacc34a86ac581f7a0a559432a"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "e7fc97d83013c9ff", "unit": "5ab444e64297fc19065478104c7ebdabe23cc415f22adafd0cf82c98e4c5cae3"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 0, "pair": "e7fc97d83013c9ff", "unit": "6b462ea30c1afda572600a10324249223932d36a032eacafac8c837bccbdce87"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "84d6a4202a082e0d", "unit": "8175082f40d8d61d44b54579c84a808e25b89110be5d8fd82feedf7dd5e0d5ab"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "8abefca43ee4ae2c797033da31f85c17bfa7be6b35413d70b0216415c5db9a09"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "91d213946859f33d3f243c43aa8285cc6e8ecb3db1de938f32af775a70607f8e"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "a244d9f010ddb43a09e82363dfabb0e406d411fb5de08ab7d6c9ee367f2a15c3"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "d27d174d6e8e5f1682a1c42723705e89f0d18c95ecedd0567e313daa9a0aae44"} +{"family": "injection", "fold": "outer-1", "group": "mongo-express/mongo-express", "label": 0, "pair": "84d6a4202a082e0d", "unit": "f45608ed47fa11ca25093641f6021ec06076c18676abac9b809f31b3b7713de8"} +{"family": "access_control", "fold": "outer-3", "group": "moquette-io/moquette", "label": 0, "pair": "55d1ce4f29dd1583", "unit": "a877ce3462da316bcdd82ce9db1d9887e887eaa83157922edb1663413edae1d6"} +{"family": "access_control", "fold": "outer-3", "group": "moquette-io/moquette", "label": 1, "pair": "55d1ce4f29dd1583", "unit": "aee67be0ba31f99c7a24faa1bad6178c6d37361a308cf5aa10138e6ce4bbcfeb"} +{"family": "path", "fold": "outer-4", "group": "naorsabag/openhop", "label": 1, "pair": "16e529fe8c5486b2", "unit": "1d21acd0171726775d3da4589b660645fcd43cc026042a2b27e0e0e8374eceb7"} +{"family": "path", "fold": "outer-4", "group": "naorsabag/openhop", "label": 0, "pair": "16e529fe8c5486b2", "unit": "f8e004b67ac98dc3392ea4115051e6bd10ce22eb638a762f0f16e1d4200bf129"} +{"family": "access_control", "fold": "outer-1", "group": "nextauthjs/next-auth", "label": 1, "pair": "0b34438e6e0ba63d", "unit": "bf2608b00f0f784cc42e4c607f0b1ef50903283779bfb2eec91d2ec187856d14"} +{"family": "access_control", "fold": "outer-1", "group": "nextauthjs/next-auth", "label": 0, "pair": "0b34438e6e0ba63d", "unit": "c5104cbf723fa0d55675963d8aa6802780ce3d274efe0012290fc49164c3fcc4"} +{"family": "output_encoding", "fold": "outer-3", "group": "nuxt-content/mdc", "label": 0, "pair": "b0210cd069757ad2", "unit": "d1d40c8b48d039c1999ff957a015273c30652ca0c45c4eb088052991a2a92f28"} +{"family": "output_encoding", "fold": "outer-3", "group": "nuxt-content/mdc", "label": 1, "pair": "b0210cd069757ad2", "unit": "e2e804652a885f640ebf341627f3279d6c965b47ee2ce792e55045ab4d1e1dee"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "nuxt-modules/og-image", "label": 1, "pair": "ef603436079b0005", "unit": "77f843b5044bb5da935df4b2620bf2abe9e7dceb3147439e883927112eaec22f"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "nuxt-modules/og-image", "label": 0, "pair": "ef603436079b0005", "unit": "f2ce95f0e44e73986784b63a702ffa4035c423056cac3667a63140a7e8d795c2"} +{"family": "injection", "fold": "outer-2", "group": "nuxt/nuxt", "label": 0, "pair": "a023056d2704c727", "unit": "6028fe87bea019646fff2b31c1b2d7bd0a95eb9214fe437e2008bc5d2dfabb7d"} +{"family": "injection", "fold": "outer-2", "group": "nuxt/nuxt", "label": 1, "pair": "a023056d2704c727", "unit": "dd93069189c5f242db611e4bfba4341c22f283b65af666fc31343854358de87e"} +{"family": "output_encoding", "fold": "outer-2", "group": "oauthlib/oauthlib", "label": 0, "pair": "abc80e34d5f72367", "unit": "87d4a5b81bbfdd9628109a5f820895b6fddb00eb96b0f7bbe30a8d640457bcc2"} +{"family": "output_encoding", "fold": "outer-2", "group": "oauthlib/oauthlib", "label": 1, "pair": "abc80e34d5f72367", "unit": "b171c77b52637bc84c487d7f8bce9a54229f9dbd6605e5089e1e88943ced989c"} +{"family": "deserialization", "fold": "outer-4", "group": "opensearch-project/opensearch-ruby", "label": 1, "pair": "1945f28a3fc5bd55", "unit": "64c0b7ccad09192eeaf590befe0de0087d5b48a65a4f6cef41415f3f54a8f536"} +{"family": "deserialization", "fold": "outer-4", "group": "opensearch-project/opensearch-ruby", "label": 0, "pair": "1945f28a3fc5bd55", "unit": "b55cd6759d4b7861e589052aa534bd0554586db46a329ed3beed7244d579e8c4"} +{"family": "injection", "fold": "outer-4", "group": "openssl/openssl", "label": 0, "pair": "1c9e47835232f644", "unit": "90e220d3540b88dd9efc8e66c8ab519d4cc78805f16bed585b795c78cfbb4d58"} +{"family": "injection", "fold": "outer-4", "group": "openssl/openssl", "label": 1, "pair": "1c9e47835232f644", "unit": "b5a185b05235f70f6cf27e2adba919f4d8393dbdf2d99de095548a21946aeaf3"} +{"family": "access_control", "fold": "outer-0", "group": "openwisp/openwisp-ipam", "label": 1, "pair": "9b2b34cf62a76ea3", "unit": "91a3984e47f17512db8523e04f863525c67db18631d8f11776cf61ca7221c91b"} +{"family": "access_control", "fold": "outer-0", "group": "openwisp/openwisp-ipam", "label": 0, "pair": "9b2b34cf62a76ea3", "unit": "d91f1e14282a0c3c8da57c6935a1625e881374bd205b5baa9e4a341d9109ae21"} +{"family": "access_control", "fold": "outer-4", "group": "oqtane/oqtane.framework", "label": 0, "pair": "a20cdf9dd208720d", "unit": "18a26df68e4b81d411cfbcc8155a7e6f382753d6e1ad95eece7ed24fa8c44324"} +{"family": "access_control", "fold": "outer-4", "group": "oqtane/oqtane.framework", "label": 1, "pair": "a20cdf9dd208720d", "unit": "e6fb74441fb585d7ef5855317fba795ef0f0a0e6eae9970a49556ce3070c3254"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "oscal-compass/compliance-trestle", "label": 0, "pair": "b2e833b60a0fa396", "unit": "08ccacb7434e9f8b8b05c7bb0996be0609e34860d8fc418db329aaf8d7910f2f"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "oscal-compass/compliance-trestle", "label": 1, "pair": "b2e833b60a0fa396", "unit": "2ca7f80f0badc049ce725718e49bd60da72d316589399126a5f4c5308cf83868"} +{"family": "path", "fold": "outer-0", "group": "owen2345/camaleon-cms", "label": 1, "pair": "d6930fe7e752b7ec", "unit": "06f0341dfbf1aa9504083737df5d9be3cedfd4e3a198799d8075a29990776800"} +{"family": "path", "fold": "outer-0", "group": "owen2345/camaleon-cms", "label": 0, "pair": "d6930fe7e752b7ec", "unit": "30e2cd8bd381d7568b78eaf982f562d278a6a8d942d5282466532c1441bdf0ad"} +{"family": "access_control", "fold": "outer-4", "group": "patriksimek/vm2", "label": 0, "pair": "e9c34a6a4bf2d333", "unit": "0ae20c7334fb27889ef8551f34c65a847dffd6b37f84f722f7e8ed64fd82e95b"} +{"family": "access_control", "fold": "outer-4", "group": "patriksimek/vm2", "label": 1, "pair": "e9c34a6a4bf2d333", "unit": "59ab1924056704605b22b7c8360f40f745e7f488450be3647a632bf9a3b22699"} +{"family": "deserialization", "fold": "outer-1", "group": "pdfminer/pdfminer.six", "label": 0, "pair": "053a9e02feda3960", "unit": "34e18fe343b5c25ec582fc05ab091fd9ba160f658ba7678066c56db4991a951b"} +{"family": "deserialization", "fold": "outer-1", "group": "pdfminer/pdfminer.six", "label": 1, "pair": "053a9e02feda3960", "unit": "8fcb18a2a8654b6594cb2ec078e2a018d20f73d2b69a2e625338ffa80b3f72c3"} +{"family": "access_control", "fold": "outer-3", "group": "perses/perses", "label": 0, "pair": "2d482951d7380c4f", "unit": "5c1a721da6f85d1d5dfc8cbe5e9809b093a09c059e5abeebed2cbbee55fbd498"} +{"family": "access_control", "fold": "outer-3", "group": "perses/perses", "label": 1, "pair": "2d482951d7380c4f", "unit": "6cabd1c4b5aa94cafdd07c0c6a3f9abfdc03761ddc5b0a2bebc72a508d020b8b"} +{"family": "injection", "fold": "outer-0", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "", "unit": "2300c097374b9098671dcbcd42d02d08172e0f3064a8323223d0fe79a6106aab"} +{"family": "injection", "fold": "outer-0", "group": "pgadmin-org/pgadmin4", "label": 1, "pair": "0664148b0b192456", "unit": "d702617aaf4d19811baaa18386a3c8b0eb35cd3fd72e8d90611d34850e990858"} +{"family": "injection", "fold": "outer-0", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "0664148b0b192456", "unit": "f5ed13d811c5b16d46cbd4a3b9edb8f7d2db49acbd49d95f39f33eaab38f58f5"} +{"family": "injection", "fold": "outer-1", "group": "phpcsstandards/php_codesniffer", "label": 0, "pair": "fd5781ba332ab81e", "unit": "59eca69b2bd99ba282eb8056d7bc789e9c691eebf62a4b42594e34f5516833ae"} +{"family": "injection", "fold": "outer-1", "group": "phpcsstandards/php_codesniffer", "label": 1, "pair": "fd5781ba332ab81e", "unit": "7a7d774a6de9e4a95fe97fff822652f1f1383922f2f0a4dcb12b8e964ca330f5"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "phpoffice/phpspreadsheet", "label": 0, "pair": "366dc5ddcfe598dd", "unit": "0053d866cec1e99f8a4a8754cf8ece3c46df16e002c86703c68937d5f3a09b08"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "phpoffice/phpspreadsheet", "label": 1, "pair": "366dc5ddcfe598dd", "unit": "027dbc5b9245fb804a1e2ba4445b8afa180aa2c2b3293db107ffe0ca4900ebb4"} +{"family": "path", "fold": "outer-1", "group": "postcss/postcss", "label": 0, "pair": "d0a34bf90e99e794", "unit": "4efbb75cc4105149522d2b2220987fd4373a8ef645c2c544546da540f14109eb"} +{"family": "path", "fold": "outer-1", "group": "postcss/postcss", "label": 1, "pair": "d0a34bf90e99e794", "unit": "fd790f533103fd1a5646a9374ae895cf6e36d578fc747db2b0d0249a54a85e9f"} +{"family": "deserialization", "fold": "outer-0", "group": "powsybl/powsybl-core", "label": 0, "pair": "a697aa902d141a6a", "unit": "341e8fbb46f00167a32a2d653ba09a205794084eff65afddb70f9cfff42d697e"} +{"family": "deserialization", "fold": "outer-0", "group": "powsybl/powsybl-core", "label": 1, "pair": "a697aa902d141a6a", "unit": "e29414494b275afc15f3a028f62402161df7f6da5123c302149e23845177f7f6"} +{"family": "output_encoding", "fold": "outer-4", "group": "prowler-cloud/prowler", "label": 0, "pair": "8c1fc8417190a134", "unit": "2125fb101d6108627e77f61c05cadddddfebd1cddce0e66376d9b822a2eecbb4"} +{"family": "output_encoding", "fold": "outer-4", "group": "prowler-cloud/prowler", "label": 1, "pair": "8c1fc8417190a134", "unit": "ae7bbc58026fb71f91b477946a27bbe3599be90501da9237755f3cf0fb289794"} +{"family": "access_control", "fold": "outer-3", "group": "publify/publify", "label": 0, "pair": "1972ea9f72b178b6", "unit": "3ddabb356d78e7c65ec8f88b6da8e4fcabc055ae73f2f6d92c50efddc5d4fcbe"} +{"family": "access_control", "fold": "outer-3", "group": "publify/publify", "label": 1, "pair": "1972ea9f72b178b6", "unit": "6875dd5e9c0ff8a1b693d29a109cb2e62e6ff4081df40868aca70e4770476757"} +{"family": "path", "fold": "outer-0", "group": "pypa/pip", "label": 1, "pair": "2b007a2377d633dd", "unit": "bdd71a783efb4eda9a82afbd30a16bd7608153830dbc599e65c453c18d1644b2"} +{"family": "path", "fold": "outer-0", "group": "pypa/pip", "label": 0, "pair": "2b007a2377d633dd", "unit": "f398ef33900624219e59de08e1cc101c43082a81c37d6d03a916cc220727ca67"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "04aa777130c02a06", "unit": "16aa43c87956c44a4b115245140eefed704095d37cec0cc6410377403cd2fa7a"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "23744137e0485a36f686f5f1b5666780fbed3ca522e8c6d35687a391ef504498"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "67565d99fe28c68f", "unit": "2692316792e6f02fb51578cf30d71bba6c73bf5420c36f709681c8eaa69807d4"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "2e36f73002c416a68d077bffb0335d82346c7507f2478353aeac005fce5a9cbc"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "04aa777130c02a06", "unit": "67721f67055eb6fed955ff22bfe4060344662eccb369431835a2116ad120c4f8"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "8fc1c96fa6faa4d3966ecc42876189b18ddab779d58f47cdf28a1155f48a6ac4"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "67565d99fe28c68f", "unit": "ab535098a8e709b022afee1da15dd1fc371d389056938ee4fa4971b2966c823a"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "d46f94235bc462360097e7a22e1d442d95f049306631489a4d8b25f647a95bbe"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "edd6dd4364442f66384d14e9317ef2f6234e32ab78327a45741b8a816b4332ee"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "fc7df6887ac719d4615934c0e8c3296728c2f6089f233158c59989027c802d72"} +{"family": "path", "fold": "outer-3", "group": "rack/rack", "label": 1, "pair": "c0412509f15fe3e1", "unit": "63bda2a71ad7155ad2f59aa9086efef3fd3710dc82c57f4b32ec8bf561460239"} +{"family": "path", "fold": "outer-3", "group": "rack/rack", "label": 0, "pair": "c0412509f15fe3e1", "unit": "b1c9a070c93764655786c3de5fde536c150e319efe47548614cc544b73a8ee2b"} +{"family": "deserialization", "fold": "outer-4", "group": "rails/kredis", "label": 1, "pair": "feed5f7de11289a4", "unit": "71531567b2b8609ae0d31cb177b8faea4f5fc61923e513b0154a2c941287ce2b"} +{"family": "deserialization", "fold": "outer-4", "group": "rails/kredis", "label": 0, "pair": "feed5f7de11289a4", "unit": "d6be6d397a301a07faa761163d3d9e270d71d23d3bdfd084eeafb1e62b91dd71"} +{"family": "access_control", "fold": "outer-1", "group": "redf0x1/camofox-mcp", "label": 1, "pair": "31c259b4e9dccd82", "unit": "1f1ddb8a5b415461638056c70756d2d0e836de71e824c0e46474f20646eaa4d5"} +{"family": "access_control", "fold": "outer-1", "group": "redf0x1/camofox-mcp", "label": 0, "pair": "31c259b4e9dccd82", "unit": "a6ec568d4eef212a842dded7fdef2284524d367e5d24473ddb34e6374be9fad7"} +{"family": "deserialization", "fold": "outer-0", "group": "redisson/redisson", "label": 1, "pair": "ffd4b04a7bebfaf8", "unit": "54f7519d3d59b6d8d69a5f8f7bd78e360d3fabaae00b7587c37e388719169370"} +{"family": "deserialization", "fold": "outer-0", "group": "redisson/redisson", "label": 0, "pair": "ffd4b04a7bebfaf8", "unit": "c5e42c9ab2ee54e46700ab0b64e239b567fbc75ccaceaf6cbf20687813663332"} +{"family": "path", "fold": "outer-3", "group": "redocly/redocly-cli", "label": 1, "pair": "2ffec9490ef1d482", "unit": "0ab883d3561162b75ad9d161f79c0d17521a23317f2f574c910465f288119483"} +{"family": "path", "fold": "outer-3", "group": "redocly/redocly-cli", "label": 0, "pair": "2ffec9490ef1d482", "unit": "b0af86c1642c14047248f1134181be0ec3839900917dffa360c5014119f8632c"} +{"family": "deserialization", "fold": "outer-2", "group": "remix-run/react-router", "label": 1, "pair": "73f2a014c6a699ed", "unit": "44ba9d9d8e75769c1444ebf37a64941a845b4bcae678ec79d64dc84e5cc067a0"} +{"family": "deserialization", "fold": "outer-2", "group": "remix-run/react-router", "label": 0, "pair": "73f2a014c6a699ed", "unit": "b430d488c486cef26b1c661f773e7e9e4a177eeea73537a033eec98d532e4eed"} +{"family": "output_encoding", "fold": "outer-3", "group": "remorses/genql", "label": 0, "pair": "738e4c22edb32172", "unit": "932d7312b2bb4b830179104bca076f587505c2d4ce881b2f04f992defba6dd9e"} +{"family": "output_encoding", "fold": "outer-3", "group": "remorses/genql", "label": 1, "pair": "738e4c22edb32172", "unit": "eee5136489c3baa3653ededbf6385d0237e2bd8b5384286780646b88ed34fb91"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "remram44/taguette", "label": 1, "pair": "cb0c638282db0d37", "unit": "a49e6ad2e7c2a5ad0d68d5d69c263306b501fc0dc9382e701fb5fe680611196b"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "remram44/taguette", "label": 0, "pair": "cb0c638282db0d37", "unit": "dee28099d3b37cbd0ee14412122ecc5f8488ed6bec01863520b87706a040fba1"} +{"family": "injection", "fold": "outer-1", "group": "roest01/node-pdf-image", "label": 0, "pair": "09bce271e8530fbc", "unit": "35eb19b4c6176b37287e2123bb98ec4045ce6bde7cc3864dde317c024b0935d8"} +{"family": "injection", "fold": "outer-1", "group": "roest01/node-pdf-image", "label": 1, "pair": "09bce271e8530fbc", "unit": "dd33291cc494bbc77d44e9b13d16f171d32c9746637506f315f375e8a9ddee95"} +{"family": "path", "fold": "outer-0", "group": "roomi-fields/notebooklm-mcp", "label": 1, "pair": "a856cd13668353a6", "unit": "6149fb0132b981620a90a23a9af8d4656d0d827403e63b48258443cacd0a9c81"} +{"family": "path", "fold": "outer-0", "group": "roomi-fields/notebooklm-mcp", "label": 0, "pair": "a856cd13668353a6", "unit": "b5aecdd6a87f8d57d28f54a013b1f96332460fe1bb44bff4ed28e166f18f9abe"} +{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 1, "pair": "40d8d81a764c508f", "unit": "7deeed73c29c74ccbed29be049e109d363d41e4b5eea245ee57780b85f0e137a"} +{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 0, "pair": "40d8d81a764c508f", "unit": "b5cdadf848e67910550666ba2a6ac67b24112e501c02dba1865766a316317a23"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "", "unit": "15dbea384b3255457a44d9382efc6944dd27f1d411d903344a5d5e5fd194b72f"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "b2ac284bd70dd684", "unit": "225890ba6694dc8f4fbab46e9c851dff8c095e906cf14db15d01ebce0891aaab"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "b6eca35de93a5be8", "unit": "42eb8427415704e060e73b5ce0943b014ddf380ca756d5ec47c558e59a79895a"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "b6eca35de93a5be8", "unit": "4c9cdcc11a18641ef41e9308712c5e8b830ec9ad5ebc3a956c8701e4f77b39c0"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "6a53939838afbdd7", "unit": "5794809e595311c86909d673a570d003aa025e4dcd6abc5641b641d935beb0b1"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "46d3ec7d49cb8ca7", "unit": "5f7a63ec650915962ee5255f5416df63ab882d6107f3b540be36d481798a8944"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "6a53939838afbdd7", "unit": "97a7018535ab511622e3ade9e53d570ffb1c80633eaa372fb76e749b3a586c8f"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "46d3ec7d49cb8ca7", "unit": "a84e9ab47f4f2b584684dbb81575758d9032fe3f2ed53bef478cb9dcfedaf677"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "b2ac284bd70dd684", "unit": "aeceb1ebc863ebbc3b69ae56c5b78d2dba005132b75cbd6af929bb0bfe25da9b"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 1, "pair": "f42cab375ace1739", "unit": "e329d4a6ac757680bf3d292a81f1f74f8a8a9b482d1651efe9f503f6a4de8796"} +{"family": "injection", "fold": "outer-0", "group": "saltcorn/saltcorn", "label": 0, "pair": "f42cab375ace1739", "unit": "e73c5aac41c0b93072223a3097fb175548ce893e0678ff3a22bc7571745a17e0"} +{"family": "injection", "fold": "outer-1", "group": "sebhildebrandt/systeminformation", "label": 1, "pair": "1041a18cd1aec5f8", "unit": "c301b8c362aee3fe5ea38e18f858166bc4162ad8ace44c5fb2df192afc81da2a"} +{"family": "injection", "fold": "outer-1", "group": "sebhildebrandt/systeminformation", "label": 0, "pair": "1041a18cd1aec5f8", "unit": "ca8e58ffcfd60bdc8626db8b9eb4f253c1b8b8affcb793c6d963b7a3ec91906b"} +{"family": "injection", "fold": "outer-4", "group": "sgabe/dsvpwa", "label": 1, "pair": "", "unit": "52354ccdbc16f209e4692d97d5237e04ed030cf4a5e71f77b5a767962b355b4a"} +{"family": "injection", "fold": "outer-4", "group": "sgabe/dsvpwa", "label": 1, "pair": "7d23be667536afff", "unit": "b5016a5a53409491ba805ed8718599b84250b7b6914da0434b0882b0b3578dc8"} +{"family": "injection", "fold": "outer-4", "group": "sgabe/dsvpwa", "label": 0, "pair": "7d23be667536afff", "unit": "f1b01102968568d3ba1b2fdc70f4ae281f433fb20d29eff2a55fc45555ddbfcd"} +{"family": "injection", "fold": "outer-0", "group": "shepherdwind/velocity.js", "label": 0, "pair": "f21df949f8aca4fa", "unit": "15d23ab3aea4c878dd73bbae579f3ab473e8363477a97d28164ed5e1b94cb510"} +{"family": "injection", "fold": "outer-0", "group": "shepherdwind/velocity.js", "label": 1, "pair": "f21df949f8aca4fa", "unit": "787bc98c651ed3db21be4c28e9c560c366ffd6ca508823d21e5d83eeaf1269db"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 0, "pair": "1199de75f426940b", "unit": "10fdc236f43a02b7e291eb8bf6ada297a2ae98086a63b171057bc7bfc37a3377"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 0, "pair": "22112c31603bbe9f", "unit": "5eb3b7523b5a196dcaacba08d68457f9a45e31503ea02745ca3a7d0542f9881c"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 1, "pair": "22112c31603bbe9f", "unit": "6bd5bdfd81b0a3374d59cfb2c1f7fa2b27887f4672f4485e77eb5234b569983f"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "signalk/signalk-server", "label": 1, "pair": "1199de75f426940b", "unit": "c09dffd04708540845c8d1dd77a2bd989bd35f264abf652a2db55f8cb1faf7a9"} +{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 0, "pair": "7aab025ea7b97019", "unit": "7487a34cdfdb532a5096764a47a3e565fa509605c1df0e2b105eb9c751fc5307"} +{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 1, "pair": "7aab025ea7b97019", "unit": "9c0be2a58741f095fd0a9cdc60d4ca4039de1529699d5701dc4d0bcf7d5bf90c"} +{"family": "path", "fold": "outer-2", "group": "simonh1000/angular-http-server", "label": 0, "pair": "4de5441f1776f083", "unit": "a3a35d573da308cd1d0614a26616bbd37b360d9ea416894cdeffdb59abd691d4"} +{"family": "path", "fold": "outer-2", "group": "simonh1000/angular-http-server", "label": 1, "pair": "4de5441f1776f083", "unit": "dc33fc26c1bf7c67e5439ddfdad06c0faf16c796e0fe8a46b6e6a541e708676d"} +{"family": "deserialization", "fold": "outer-3", "group": "skops-dev/skops", "label": 0, "pair": "1972a9a06d7585ad", "unit": "1f472e842e894c9907536c6eb6c9c922bbb3b21c27999114588ec2a4345ab665"} +{"family": "deserialization", "fold": "outer-3", "group": "skops-dev/skops", "label": 1, "pair": "1972a9a06d7585ad", "unit": "7e38e24f1cc4d287627b7cd6cb870888865bb87fa87bba1c4fbf84eabb213121"} +{"family": "access_control", "fold": "outer-2", "group": "smithyhq/sqladmin", "label": 1, "pair": "d76d5232a8587127", "unit": "1ec32d427822d1ddd3b0d7245f5771ddeb56c146ac8ff9959dd85760ff659a20"} +{"family": "access_control", "fold": "outer-2", "group": "smithyhq/sqladmin", "label": 0, "pair": "d76d5232a8587127", "unit": "5143b19a1a1e6b9162b80594ed7fb5c0c57ed0b8abfd793bfc71089552d57638"} +{"family": "injection", "fold": "outer-2", "group": "soyuka/pidusage", "label": 0, "pair": "5f6e814444b4acd8", "unit": "2e8aa07b1aa4bfc5349b0d4464f3f2a6500aa55ff765b0139e9901ed34151ffa"} +{"family": "injection", "fold": "outer-2", "group": "soyuka/pidusage", "label": 1, "pair": "5f6e814444b4acd8", "unit": "e26c726c08be761b22fcc6ec17fc1fa7aba4769f08957f024e5545906df62797"} +{"family": "output_encoding", "fold": "outer-3", "group": "spatie/schema-org", "label": 1, "pair": "888e955fb6cfb4fd", "unit": "6f1a431ca97afc34ceec48393eb134a8928a312e6f761eb907c6b5855cac6208"} +{"family": "output_encoding", "fold": "outer-3", "group": "spatie/schema-org", "label": 0, "pair": "888e955fb6cfb4fd", "unit": "7b62415d17569924d16238a4aa28ab1af2db6561f56be9f59d8075fd1bc4a941"} +{"family": "injection", "fold": "outer-2", "group": "spring-projects/spring-data-mongodb", "label": 1, "pair": "a5691395e641fd4c", "unit": "40339c123107f1231d6aa58de876cded1dfefd5b31dadc4fc86eedc6a687165f"} +{"family": "injection", "fold": "outer-2", "group": "spring-projects/spring-data-mongodb", "label": 0, "pair": "a5691395e641fd4c", "unit": "ea3828e2f59cc6f7b415a101532b566ecbd47ab123c3c7593c23c52f6773a2e3"} +{"family": "access_control", "fold": "outer-0", "group": "spring-projects/spring-graphql", "label": 1, "pair": "f1ebdacbd647e39d", "unit": "79628ef3d6c9e12a94922cdca548ff77d3dbdd6b595c1fec5958fcf23b609095"} +{"family": "access_control", "fold": "outer-0", "group": "spring-projects/spring-graphql", "label": 0, "pair": "f1ebdacbd647e39d", "unit": "f910e010c3ff2c8c17f0f80fbe89a934e4f792b601febbf318359e6dc3afc18e"} +{"family": "output_encoding", "fold": "outer-4", "group": "squidfunk/mkdocs-material", "label": 0, "pair": "2dd5384f097711b5", "unit": "212fba6b5e50db045114af369a03ffbdf09ec898c10eeff3c45bfc01a5017b4c"} +{"family": "output_encoding", "fold": "outer-4", "group": "squidfunk/mkdocs-material", "label": 1, "pair": "2dd5384f097711b5", "unit": "370ba3a1d0e354e882b7be92c1ebbf2e2903bdf29d442048ea3dc1f80922467f"} +{"family": "injection", "fold": "outer-4", "group": "strangerstudios/paid-memberships-pro", "label": 1, "pair": "", "unit": "ac905d12b92954263c046bd0138a682cea86cf8476336d6baa664ade28d34ac6"} +{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 1, "pair": "218a45d869bd7e39", "unit": "309ebac468b7ab054c8396ed495770385ae3590caba43709745ba2d1db4b6d54"} +{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 1, "pair": "", "unit": "3cc8613813dec13cb1b7b45b296688da883f0b36cd2c4fc1da6e81703fb59ab9"} +{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 0, "pair": "218a45d869bd7e39", "unit": "9bd2331dbc260451355307e4c37bae17ad6676654c66181934e1b35fb174cc5a"} +{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "b087ac4d0989e0b72698a9644ef4eac8073b4a15bb21730d63e4730b5cc58a60"} +{"family": "injection", "fold": "outer-0", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "c5d1e8d19560c997018fdca12eb60f1799be76c33be058b88472f5a2fc6709be"} +{"family": "output_encoding", "fold": "outer-2", "group": "svg/svgo", "label": 0, "pair": "c863054be488a284", "unit": "425e1d9961b79bbc011c447cd050910dd796a32b248962a5b0acc3f5f19b7345"} +{"family": "output_encoding", "fold": "outer-2", "group": "svg/svgo", "label": 1, "pair": "c863054be488a284", "unit": "552925bba199119c7db5457b9d3c535bc9f6061cecfd2ab2f72cde690a536e1e"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "22befc17a75a688e2f9358d2ddf9a7b197da44cd95dbdeae5e0191b113ee54d3"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 0, "pair": "63f496d9ff53e39b", "unit": "6a98f8145776c78793b37156e9a81757841bb6de80016ab4fa575209d724db3f"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 1, "pair": "63f496d9ff53e39b", "unit": "7ac60c96c15f8517e089cc2240aedcf1fc30e54a41e35fe03c6b965bc393fbd6"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "ae39cb55ef2a87c5347e0ddc40975f142450e82de4909e93ba62a7ed0efb4134"} +{"family": "injection", "fold": "outer-3", "group": "thenables/thenify", "label": 1, "pair": "d1b19898f8deaa41", "unit": "3763c7b407ad471820b5f8fa25bde0ba6d91e15c07c56681de4b41add418b0b1"} +{"family": "injection", "fold": "outer-3", "group": "thenables/thenify", "label": 0, "pair": "d1b19898f8deaa41", "unit": "bc8b934938c30687f3d4f982392d2cfa9504a916b463ccb5c46fe8a30eede488"} +{"family": "path", "fold": "outer-3", "group": "thumbor/thumbor", "label": 1, "pair": "f5f33fa6fe7b8492", "unit": "938bbf146a9037e687190bcee82c047492dffbe90efea40a49a647f81a0f1ed0"} +{"family": "path", "fold": "outer-3", "group": "thumbor/thumbor", "label": 0, "pair": "f5f33fa6fe7b8492", "unit": "b90c9f7e0011685f1a8e853a9ed0ae37fd437ef6d6eb0ba17c240aec2c22b603"} +{"family": "deserialization", "fold": "outer-2", "group": "tomasnorre/crawler", "label": 0, "pair": "f74364dc8e35c8ba", "unit": "11e760e99c5c852676c9288e18a250febb3854e9047b51e8b9ecc9f3b423b6f5"} +{"family": "deserialization", "fold": "outer-2", "group": "tomasnorre/crawler", "label": 1, "pair": "f74364dc8e35c8ba", "unit": "5fc980194040f4c4d803075d4085618cc0260ae8f3454b26b9b704d30200ffea"} +{"family": "output_encoding", "fold": "outer-1", "group": "tornadoweb/tornado", "label": 0, "pair": "db1c51833994ec55", "unit": "c2ce8320c778dd55434d7413844797b68e6fcabc4dc509458a9d60ca70b88c60"} +{"family": "output_encoding", "fold": "outer-1", "group": "tornadoweb/tornado", "label": 1, "pair": "db1c51833994ec55", "unit": "f2d6843547af1276ae8a4dff3a7786913fa79dc69540d4fbb95c4c091e27c7f2"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "truelayer/truelayer-dotnet", "label": 1, "pair": "f4690679bf9d80aa", "unit": "1a4e24df8d08672ecde4709a77ea5c437b2198fecff8bfff373b63c9eaed6e63"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "truelayer/truelayer-dotnet", "label": 0, "pair": "f4690679bf9d80aa", "unit": "78711973ec84867671314dd7878769092b32f59860eafed50a0f6979daee76fc"} +{"family": "injection", "fold": "outer-2", "group": "typeorm/typeorm", "label": 1, "pair": "d6a42c18151efb48", "unit": "01ae2279707361bfc06a005335d41b85066d0a163802ca81c031f5dbded5bb46"} +{"family": "injection", "fold": "outer-2", "group": "typeorm/typeorm", "label": 0, "pair": "d6a42c18151efb48", "unit": "a50dc15c313a9078badea01365987c66c5cee1ad48cea65c35c8bc2cbba25e8e"} +{"family": "injection", "fold": "outer-3", "group": "ultimatemember/ultimatemember", "label": 0, "pair": "85c15e3fc2b370b2", "unit": "0990d8f41cb07c1cf217812e51a7025381de4ba68d1be3799e26241d81dcea67"} +{"family": "injection", "fold": "outer-3", "group": "ultimatemember/ultimatemember", "label": 1, "pair": "85c15e3fc2b370b2", "unit": "9da1d63f0c4da270f66eb87bf9c342ace883a0aef78c4643e254159b5da8fce3"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "unjs/ufo", "label": 1, "pair": "c7f4f4ebebfae831", "unit": "b5cbc1490bca76679adfbe5e4d9f8067d27a226d595ba401b551b00c3ce4d45a"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "unjs/ufo", "label": 0, "pair": "c7f4f4ebebfae831", "unit": "ba2c1808eebdbe3f54c488b09bcb3d69e521916d1a3a0b5b14e02d5bc88bf016"} +{"family": "path", "fold": "outer-1", "group": "vercel/next.js", "label": 1, "pair": "1507c8e00ed4b859", "unit": "620e9b5ed143c0bcec178db359fa601acb1d3a134b3e85c2e838b7dcf3892d85"} +{"family": "path", "fold": "outer-1", "group": "vercel/next.js", "label": 0, "pair": "1507c8e00ed4b859", "unit": "6f5ce4008dac0471260310c60e1e2a9f76a7e4eec6ebd039835f0488cf3b19aa"} +{"family": "output_encoding", "fold": "outer-1", "group": "viewcomponent/view_component", "label": 0, "pair": "8f5a6abedd742bbe", "unit": "71e3ced643a244100cc3fde234f5b4b27655214c7ae0ff67ab61e3e48a8fdf78"} +{"family": "output_encoding", "fold": "outer-1", "group": "viewcomponent/view_component", "label": 1, "pair": "8f5a6abedd742bbe", "unit": "7c8cbfae87127042f9f43621345b4f7958bde27a3ff9db76254063e80735f3ce"} +{"family": "injection", "fold": "outer-3", "group": "vincentmorneau/apex-publish-static-files", "label": 1, "pair": "c9538342f2d7efca", "unit": "2b8db57d65ef1d67c457ff5a336e8a542d03387395d69d5e16815c4f00f40723"} +{"family": "injection", "fold": "outer-3", "group": "vincentmorneau/apex-publish-static-files", "label": 0, "pair": "c9538342f2d7efca", "unit": "e2ed113dc21ad5f7caa71ba4a5ece8f8f900a97fd0d00dc85f7ac79ff5bed19a"} +{"family": "path", "fold": "outer-2", "group": "vitest-dev/vitest", "label": 0, "pair": "494dd5d5e5cdf4b8", "unit": "3b8011f94ec721f68dd29afc6c8b01117c86a8b0c31f3650a6ce0a8d7c0739ff"} +{"family": "path", "fold": "outer-2", "group": "vitest-dev/vitest", "label": 1, "pair": "494dd5d5e5cdf4b8", "unit": "f7e2280574e9ae51daa64d8adb2955407ff6d1e156f0e2854b6c047e0a733c78"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "vllm-project/vllm", "label": 1, "pair": "51eae488182a123a", "unit": "20b039866ba796c34efa0e92468fb49a314ad7fe2914d9ff8b823e3e35ab26f9"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "vllm-project/vllm", "label": 0, "pair": "51eae488182a123a", "unit": "2ab3fef05def768ef7b4165bcedcf7d200282dba78a32e19b90a0df3aee7e3e5"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "vuejs/vue-cli", "label": 1, "pair": "6f5a650865e08571", "unit": "e2051cc42e4f8b026c7807f966d492367310c90326e60fef1b0ee8c509c805a9"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "vuejs/vue-cli", "label": 0, "pair": "6f5a650865e08571", "unit": "f5364d68ffe80b578e788f37f08784b0162826393b49b4a3affd70d7cc4c9e01"} +{"family": "injection", "fold": "outer-2", "group": "vvo/node-whereis", "label": 1, "pair": "8acd426c5771b37c", "unit": "983e146a60e061254b15dc8dd573cbba57f8dd0358f23ea1b9c313ef69069af5"} +{"family": "injection", "fold": "outer-2", "group": "vvo/node-whereis", "label": 0, "pair": "8acd426c5771b37c", "unit": "f80dfec45231e4778078cb10f00128ed445177cac0a062fc2df5ced5c0171f21"} +{"family": "deserialization", "fold": "outer-2", "group": "waterfutures/epyt-flow", "label": 0, "pair": "9b84800ce5c395cb", "unit": "9b7dfdceae03fd86ad3260a03f85bcdcff8f32f10c265d23743ff8dc98eeefb8"} +{"family": "deserialization", "fold": "outer-2", "group": "waterfutures/epyt-flow", "label": 1, "pair": "9b84800ce5c395cb", "unit": "b99dda150c0da3defa5f4c527ef6c402f5589dfb2f2f03f3e5e424910eb48c53"} +{"family": "path", "fold": "outer-1", "group": "webpack/webpack-dev-middleware", "label": 1, "pair": "ad64e8fac4ac3ee0", "unit": "36dfd01915a553d86576e8ecd3b737b8e2b297aa542f58cc83b448060e74398a"} +{"family": "path", "fold": "outer-1", "group": "webpack/webpack-dev-middleware", "label": 0, "pair": "ad64e8fac4ac3ee0", "unit": "c2b6d40f130d35bbfc16fab9411c664fa06321c59fb1d901a1987a69cb09c72e"} +{"family": "access_control", "fold": "outer-0", "group": "wintercms/winter", "label": 0, "pair": "d48c08b19f560848", "unit": "03d02c9e25eec0f380b125279fce35e8867065bf6e1ec75f9f39729ce6efdc77"} +{"family": "access_control", "fold": "outer-0", "group": "wintercms/winter", "label": 1, "pair": "d48c08b19f560848", "unit": "abbea69b7d0fa68c3ace442c25597e128cde354b417787b5913fd10d666c0d19"} +{"family": "output_encoding", "fold": "outer-2", "group": "xdan/jodit", "label": 0, "pair": "bfc8188bbb673e3b", "unit": "a7cff0da55c0cc1d5c838e11fc6ce6c4b22c54b9f8e101688b218d6928eca504"} +{"family": "output_encoding", "fold": "outer-2", "group": "xdan/jodit", "label": 1, "pair": "bfc8188bbb673e3b", "unit": "f741c56815230b9c30e2c8ef8952b895e9480e4780a89ff2cf6daf60d1fce7b6"} +{"family": "output_encoding", "fold": "outer-4", "group": "xyproto/algernon", "label": 0, "pair": "3844ed3d703d7ef4", "unit": "445efda074e4d9f6c1634225eb08a3c3eb1257311c995ce891df2c791a6f5c8b"} +{"family": "output_encoding", "fold": "outer-4", "group": "xyproto/algernon", "label": 1, "pair": "3844ed3d703d7ef4", "unit": "8ac27b08549353ada819caaf99a5180d2f2f515b03aa8cd99a5d37b527ea3a5c"} +{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 1, "pair": "a63eae7d621a985e", "unit": "04ca0b583b09c3b4f67bd2496a4607ecb134d24b7bdaf8960fa151e40ebcc447"} +{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 0, "pair": "a63eae7d621a985e", "unit": "ecfa2a67726ca9d3e2621a4d8be60145095b76aa18ecf0268212341d63752871"} +{"family": "deserialization", "fold": "outer-1", "group": "zumba/json-serializer", "label": 0, "pair": "0d9f529b7f0084b6", "unit": "35aa369236f510634b3aa7b54b04e9da0f80877e515786a888e760f8fe33c524"} +{"family": "deserialization", "fold": "outer-1", "group": "zumba/json-serializer", "label": 1, "pair": "0d9f529b7f0084b6", "unit": "ef4f8613c0914eb815504e00cd6ee05df38352fd3b27826798e6623594017042"} diff --git a/src/openultrasast/learn/cli.py b/src/openultrasast/learn/cli.py index fe3e4aa..968cc1c 100644 --- a/src/openultrasast/learn/cli.py +++ b/src/openultrasast/learn/cli.py @@ -60,18 +60,24 @@ def add_commands(learn_sub: Any) -> None: ("register", "record the manifest's and the units file's sha256 as the experiment row (both committed at HEAD)"), ("units", "freeze the units of a program experiment from the memory (no model): the families' evaluation candidates"), ("run", "run every arm on every unit, paired; arm A replays from the response cache when the manifest says so"), + ("fill", "fill a rule experiment's response cache under a separate budget, or estimate with --dry-run"), ("analyse", "paired cluster bootstrap, exact McNemar, the two looks and the adoption verdict (counts and intervals only)"), ): sub = experiment_sub.add_parser(name, help=text) sub.add_argument("manifest", type=Path, help="plane/experiments/.yaml") sub.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)") - if name in ("units", "run"): + if name in ("units", "run", "fill"): sub.add_argument("--profile", choices=("static", "plane"), default="static") - if name == "run": + if name in ("run", "fill"): sub.add_argument( "--manifest-model", type=Path, default=Path("plane/models/deepseek-flash.yaml"), help="the chat Model (prices)" ) + if name == "run": sub.add_argument("--no-early-stop", action="store_true", help="take no first look during the run") + if name == "fill": + sub.add_argument("--budget-usd", type=float, help="required for paid fill; includes the smoke call") + sub.add_argument("--dry-run", action="store_true", help="inspect and estimate only; no calls or writes, including --out") + sub.add_argument("--out", type=Path, help="also write the fill report as JSON (ignored for --dry-run)") if name in ("run", "analyse"): sub.add_argument("--resamples", type=int, help="bootstrap resamples (default: the manifest's)") sub.add_argument("--out", type=Path, help="write the counts-only report here (analyse --record: benchmarks/experiments//)") @@ -154,6 +160,33 @@ def _experiment(args: argparse.Namespace) -> int: data = store.get_blob("excerpts", sha) return data.decode("utf-8") if data is not None else None + if args.experiment_command == "fill": + from ..tool_hunter import ChatClient + from .fill import fill_rule + + def resolve_fill_client() -> ChatClient: + from ..config import load_config + from ..model.endpoint import resolve_chat_endpoint + + resolved = resolve_chat_endpoint(load_config(None)) + if resolved is None: + raise ex.ExperimentError("no chat endpoint configured; nothing was spent") + return resolved[0] + + report = fill_rule( + store, + manifest, + memory, + parameters, + excerpt_text, + vectors, + client_factory=resolve_fill_client, + budget_usd=args.budget_usd, + dry_run=args.dry_run, + ) + _write(report, None if args.dry_run else args.out) + return {"done": 0, "dry_run": 0, "unfinished": 3, "failed": 1}[report["status"]] + if manifest.kind == "rule": # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store) summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors) diff --git a/src/openultrasast/learn/experiments.py b/src/openultrasast/learn/experiments.py index 24947bd..803ba0b 100644 --- a/src/openultrasast/learn/experiments.py +++ b/src/openultrasast/learn/experiments.py @@ -35,6 +35,7 @@ import math import random import subprocess from collections.abc import Callable, Iterable, Mapping, Sequence +from contextlib import suppress from dataclasses import asdict, dataclass, field, fields, replace from pathlib import Path from typing import Any @@ -340,8 +341,22 @@ def registration_id(experiment_id: str) -> str: def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None: + """Return the authoritative registration, preferring a set units digest over pending rows. + + Within that group, use the greatest ``created`` value when all values are nonempty and mutually + comparable (strings compare lexically). Otherwise, or on a tie, use the lexically greatest ``pin``, + then the last row in store order (key/id order for MemoryStore, insertion order for an append-only store). + """ rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK] - return rows[0] if rows else None + if not rows: + return None + rows = [r for r in rows if r.get("units_digest")] or rows + rows = sorted(rows, key=lambda r: str(r.get("pin", ""))) + if all(r.get("created") for r in rows): + # Incomparable timestamps retain the deterministic pin/store ordering. + with suppress(TypeError): + rows = sorted(rows, key=lambda r: r["created"]) + return rows[-1] def register( diff --git a/src/openultrasast/learn/fill.py b/src/openultrasast/learn/fill.py new file mode 100644 index 0000000..36d9139 --- /dev/null +++ b/src/openultrasast/learn/fill.py @@ -0,0 +1,207 @@ +"""Fill rule-experiment responses without writing scores or changing the frozen population. + +Estimates use the family's mean billed cached response cost when usage is available. +Otherwise they use Prompt.estimated_tokens and 300 output tokens. Unknown answers may require one retry; +dry-run counts those separately as conditional requests, never fabricating an answer. +""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from ..model.endpoint import DeepSeekChatClient +from ..plane.budget import BudgetExhausted, MeteredClient, cost_of, prices_from +from ..plane.memory import MemoryStore +from ..tool_hunter import ChatClient, ChatResponse +from . import experiments as ex +from .examples import Example +from .program import Caller, Program, Prompt, parse_answer, request_key + +DEFAULT_OUTPUT_TOKENS = 300 +DEFAULT_MARGIN_USD = 0.01 + + +@dataclass(frozen=True) +class Request: + family: str + prompt: Prompt + temperature: float + sample: str + + def key(self, caller: Caller) -> str: + return request_key(caller.model, caller.params_digest, self.prompt.messages, self.temperature, self.sample, True) + + def ask(self, caller: Caller) -> str: + return caller.ask(self.prompt.messages, temperature=self.temperature, sample=self.sample) + + def retry(self) -> Request: + return Request(self.family, self.prompt, 0.0, f"{self.sample}:retry") + + +class FillMeter(MeteredClient): + """Reserve a whole call before admission, including the first (smoke) call. + + The fixed cent is a floor, never replaced by a smaller observed cost. For the + production adapter, UTF-8 bytes conservatively bound input tokens (plus 256 framing + tokens per message), and the provider enforces the output limit. Injected clients + must honour the same token/usage contract. No hidden transport retries are allowed. + """ + + def __init__(self, client: ChatClient, parameters: Mapping[str, Any], budget: float, output_limit: int) -> None: + super().__init__(client, prices=parameters, budget_usd=budget) + self.limit = budget + self.output_limit = output_limit + self.largest_call_usd = 0.0 + + def complete(self, **kwargs: Any) -> ChatResponse: + assert self.prices is not None + messages = kwargs["messages"] + input_bound = sum(len(str(m["content"]).encode("utf-8")) + 256 for m in messages) + bound = ( + input_bound * max(self.prices.input_per_m, self.prices.cache_hit_per_m) + self.output_limit * self.prices.output_per_m + ) / 1_000_000 + reserve = max(DEFAULT_MARGIN_USD, self.largest_call_usd, bound) + self.budget_usd = max(0.0, self.limit - reserve) + before = self.usd or 0.0 + tokens_before = sum(self.usage.values()) + try: + response = super().complete(**kwargs) + except BudgetExhausted: + raise + except Exception: + self.calls += 1 # attempted provider calls count even when no response was returned + raise + finally: + self.largest_call_usd = max(self.largest_call_usd, (self.usd or 0.0) - before) + if sum(self.usage.values()) <= tokens_before: + raise RuntimeError("provider returned no metered token usage") + return response + + +def fill_rule( + store: MemoryStore, + manifest: ex.Manifest, + memory: Sequence[Example], + parameters: Mapping[str, Any], + excerpt_text: Callable[[str], str | None], + vectors: Mapping[str, Sequence[float]] | None = None, + *, + client: ChatClient | None = None, + client_factory: Callable[[], ChatClient] | None = None, + budget_usd: float | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + """Inspect or fill every frozen unit. The first cache miss is the smoke call. + + A successful smoke is retained in the cache; any provider error stops immediately + and is reported without its message, which may contain credentials. A factory is + resolved only after validation, and only when a paid request is actually missing. + """ + from .evaluate import candidate_from_example + + if manifest.kind != "rule": + raise ex.ExperimentError("experiment fill supports kind: rule only") + if not dry_run and (budget_usd is None or not math.isfinite(budget_usd) or budget_usd < 0): + raise ex.ExperimentError("--budget-usd must be a finite non-negative number (or use --dry-run)") + prices = prices_from(parameters) + if prices is None or any(not math.isfinite(p) or p < 0 for p in (prices.input_per_m, prices.output_per_m, prices.cache_hit_per_m)): + raise ex.ExperimentError("fill requires finite, non-negative Model prices") + ex.check_registered(store, manifest) + units, folds, _ = ex._checked_units(manifest, memory) + programs = {f: Program(ex.arm_spec(manifest, manifest.arms["A"], f, store), memory, excerpt_text, vectors) for f in manifest.families} + models = {p.spec.model for p in programs.values()} + if len(models) != 1: + raise ex.ExperimentError("the rule programs must use the same model") + caller = Caller(None, next(iter(models)), parameters, store=store) + index = {e.id: e for e in memory} + candidate_of = candidate_from_example(excerpt_text, vectors) + requests: list[Request] = [] + for unit in sorted(units, key=lambda u: (u.group, u.family, u.unit)): + candidate = candidate_of(index[unit.unit]) + if candidate is None: + raise ex.ExperimentError(f"{manifest.id}: unit {unit.unit[:12]} has no excerpt in the store") + for prompt, _, temperature, sample in programs[unit.family].prepare_samples( + candidate, folds[unit.fold], seed=int(manifest.folds.get("seed", 0)) + ): + requests.append(Request(unit.family, prompt, temperature, sample)) + + # Request counts are distinct cache keys per family, even when multiple units share one. + def inventory() -> dict[str, dict[str, Request]]: + result: dict[str, dict[str, Request]] = {f: {} for f in manifest.families} + for req in requests: + result[req.family][req.key(caller)] = req + entry = caller._cached(req.key(caller)) + if entry is not None and parse_answer(entry.get("content"), req.prompt.valid_lines) is None: + retry = req.retry() + result[req.family][retry.key(caller)] = retry + return result + + initial = inventory() + initially_cached = {key for cells in initial.values() for key in cells if caller._cached(key) is not None} + families: dict[str, Any] = {} + for family, cells in initial.items(): + usages = [ + usage + for key in cells + if (entry := caller._cached(key)) is not None and isinstance(usage := entry.get("usage"), Mapping) and usage + ] + outputs = [float(usage["completion_tokens"]) for usage in usages if "completion_tokens" in usage] + mean_output = sum(outputs) / len(outputs) if outputs else DEFAULT_OUTPUT_TOKENS + missing = [req for key, req in cells.items() if key not in initially_cached] + families[family] = { + "units": sum(u.family == family for u in units), + "requests": len(cells), + "cached": len(cells) - len(missing), + "missing": len(missing), + "filled": 0, + "still_missing": len(missing), + "estimate_method": "measured" if usages else "estimated", + "estimated_usd": len(missing) * sum(cost_of(usage, prices) for usage in usages) / len(usages) + if usages + else sum((r.prompt.estimated_tokens * prices.input_per_m + mean_output * prices.output_per_m) / 1_000_000 for r in missing), + "estimated_output_tokens": mean_output, + "conditional_retries": sum(not r.sample.endswith(":retry") for r in missing), + } + report: dict[str, Any] = { + "experiment": manifest.id, + "status": "dry_run" if dry_run else "done", + "families": families, + "client_calls": 0, + "usd": 0.0, + "reason": "", + } + if dry_run or not any(c["missing"] for c in families.values()): + return report + meter: FillMeter | None = None + try: + if client is None and client_factory is not None: + client = client_factory() + if client is None: + raise ex.ExperimentError("no chat endpoint configured; nothing was spent") + output_limit = max(p.spec.max_output_tokens for p in programs.values()) + if isinstance(client, DeepSeekChatClient): + client = client.bounded(output_limit) + assert budget_usd is not None + meter = FillMeter(client, parameters, budget_usd, output_limit) + caller.client = meter + for req in requests: + content = req.ask(caller) + if parse_answer(content, req.prompt.valid_lines) is None: + req.retry().ask(caller) + except BudgetExhausted: + report.update(status="unfinished", reason="budget exhausted; resume with a new fill budget") + except ex.ExperimentError: + raise + except Exception: + # Never echo provider exceptions: response bodies and URLs may contain secrets. + report.update(status="failed", reason="provider or cache failure; fill stopped") + final = inventory() + for family, cells in final.items(): + cached = {key for key in cells if caller._cached(key) is not None} + families[family].update(requests=len(cells), filled=len(cached - initially_cached), still_missing=len(cells) - len(cached)) + if meter is not None: + report.update(client_calls=meter.calls, usd=meter.usd) + return report diff --git a/src/openultrasast/learn/program.py b/src/openultrasast/learn/program.py index 436cc68..dd182a7 100644 --- a/src/openultrasast/learn/program.py +++ b/src/openultrasast/learn/program.py @@ -506,20 +506,24 @@ class Program: out.append((prompt, got)) return out - def decide(self, candidate: Candidate, fold: Fold, caller: Caller, *, seed: int = 0) -> Decision: + def prepare_samples(self, candidate: Candidate, fold: Fold, *, seed: int = 0) -> list[tuple[Prompt, Retrieval, float, str]]: + """The exact prompts, temperatures and sample labels shared by execution and cache planning.""" if self.spec.sampling == RETRIEVAL_ENSEMBLE: prepared = self.prepare_ensemble(candidate, fold, seed=seed) - answers = [ - sample_answer(prompt, caller, temperature=TEMPERATURES[0], sample=str(j), families=self.families) - for j, (prompt, _) in enumerate(prepared) - ] - got = prepared[0][1] - return replace(aggregate(answers), retrieval=got.mode, neighbours=got.neighbours) + return [(prompt, got, TEMPERATURES[0], str(j)) for j, (prompt, got) in enumerate(prepared)] if self.spec.sampling != SAMPLING[0]: raise ValueError(f"unknown sampling {self.spec.sampling!r} (one of {', '.join(SAMPLING)})") prompt, got = self.prepare(candidate, fold, seed=seed) - decision = classify(prompt, caller, self.spec.k, families=self.families) - return replace(decision, retrieval=got.mode, neighbours=got.neighbours) + return [(prompt, got, TEMPERATURES[0] if j == 0 else TEMPERATURES[1], str(j)) for j in range(self.spec.k)] + + def decide(self, candidate: Candidate, fold: Fold, caller: Caller, *, seed: int = 0) -> Decision: + prepared = self.prepare_samples(candidate, fold, seed=seed) + answers = [ + sample_answer(prompt, caller, temperature=temperature, sample=sample, families=self.families) + for prompt, _, temperature, sample in prepared + ] + got = prepared[0][1] + return replace(aggregate(answers), retrieval=got.mode, neighbours=got.neighbours) __all__ = [ diff --git a/src/openultrasast/model/endpoint.py b/src/openultrasast/model/endpoint.py index f0c9332..167cad5 100644 --- a/src/openultrasast/model/endpoint.py +++ b/src/openultrasast/model/endpoint.py @@ -96,6 +96,21 @@ class DeepSeekChatClient: self._disable_thinking = disable_thinking self.endpoint = endpoint self.usage: list[dict[str, object]] = [] # one entry per call, for the round's cost meter + self._output_limit: int | None = None + + def bounded(self, max_output_tokens: int) -> DeepSeekChatClient: + """A single-attempt, output-capped copy for callers reserving a hard per-call budget. + + Empty JSON retries belong to the caller's meter, and transport failures must + stop that caller before another potentially billable attempt. + """ + from dataclasses import replace + + if max_output_tokens <= 0: + raise ValueError("max_output_tokens must be positive") + client = DeepSeekChatClient(replace(self._client, max_attempts=1), disable_thinking=self._disable_thinking, endpoint=self.endpoint) + client._output_limit = max_output_tokens + return client def complete( self, @@ -117,7 +132,7 @@ class DeepSeekChatClient: logprobs=logprobs, temperature=temperature, ) - if json_object and not (response.content or "").strip(): + if self._output_limit is None and json_object and not (response.content or "").strip(): response = self._call( model=model, messages=messages, @@ -147,6 +162,8 @@ class DeepSeekChatClient: temperature: float | None = None, ) -> ChatResponse: extra: dict[str, object] = {} + if self._output_limit is not None: + extra["max_tokens"] = self._output_limit if self._disable_thinking: extra["thinking"] = {"type": "disabled"} # otherwise temperature is silently ignored if temperature is not None: diff --git a/tests/test_learn_experiments.py b/tests/test_learn_experiments.py index 0e23844..c9efa7c 100644 --- a/tests/test_learn_experiments.py +++ b/tests/test_learn_experiments.py @@ -1,13 +1,14 @@ """A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule, arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores -are ``FileStore`` under ``tmp_path``.""" +are in-memory fakes or ``FileStore`` under ``tmp_path``.""" from __future__ import annotations import json import math from pathlib import Path +from unittest.mock import Mock import pytest from learn_fixtures import ScriptedChat, corpus, x_record @@ -18,7 +19,7 @@ from openultrasast.learn.compile import canonical, program_id from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec from openultrasast.learn.retrieve import Target from openultrasast.plane.budget import MeteredClient -from openultrasast.plane.memory import FileStore +from openultrasast.plane.memory import FileStore, MemoryStore, Record ROOT = Path(__file__).resolve().parents[1] PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32} @@ -170,6 +171,91 @@ def test_register_refuses_uncommitted_and_modified_manifests(tmp_path: Path) -> assert len(store.rows(kind="experiment")) == 1 +@pytest.fixture +def registration_store() -> Mock: + records: list[Record] = [] + store = Mock(spec=MemoryStore) + store.put_row.side_effect = lambda row: records.append(Record(dict(row), "memory", None)) + store.rows.side_effect = lambda *, repo, kind: [r for r in records if r.row["repo"] == repo and r.row["kind"] == kind] + return store + + +def test_registered_prefers_frozen_over_pending(tmp_path: Path, registration_store: Mock) -> None: + manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64)) + pending = ex.register(registration_store, manifest, commit_of=committed, created="2026-10-03") + ex.write_units(manifest.units_file, [unit("u", "g", 1)]) + frozen = ex.register(registration_store, manifest, commit_of=lambda _: PIN, created="2026-10-02") + assert pending["units_status"] == "pending" + assert len(registration_store.rows(repo=ex.experiment_repo(manifest.id), kind=ex.EXPERIMENT_KIND)) == 2 + assert ex.registered(registration_store, manifest.id) == frozen + assert ex.check_registered(registration_store, manifest, need_units=True) == frozen + + +def test_registered_frozen_only(tmp_path: Path, registration_store: Mock) -> None: + manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64)) + ex.write_units(manifest.units_file, [unit("u", "g", 1)]) + frozen = ex.register(registration_store, manifest, commit_of=committed) + assert ex.registered(registration_store, manifest.id) == frozen + assert ex.check_registered(registration_store, manifest, need_units=True) == frozen + + +def test_registered_pending_only_refuses_units(tmp_path: Path, registration_store: Mock) -> None: + manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64)) + pending = ex.register(registration_store, manifest, commit_of=committed) + ex.write_units(manifest.units_file, [unit("u", "g", 1)]) + assert ex.registered(registration_store, manifest.id) == pending + with pytest.raises(ex.ExperimentError, match=r"the units file was not registered \(pending\)"): + ex.check_registered(registration_store, manifest, need_units=True) + + +@pytest.mark.parametrize("freeze", [False, True]) +def test_register_still_refuses_different_manifest_digest(tmp_path: Path, registration_store: Mock, freeze: bool) -> None: + path = write_manifest(tmp_path, "0" * 64) + manifest = ex.load_manifest(path) + ex.register(registration_store, manifest, commit_of=committed) + if freeze: + ex.write_units(manifest.units_file, [unit("u", "g", 1)]) + ex.register(registration_store, manifest, commit_of=lambda _: PIN) + path.write_text(path.read_text().replace("mde: 0.1", "mde: 0.2")) + with pytest.raises(ex.ExperimentError, match="A changed manifest is a new experiment"): + ex.register(registration_store, ex.load_manifest(path), commit_of=committed) + assert registration_store.put_row.call_count == 1 + freeze + + +@pytest.mark.parametrize( + ("first_created", "second_created", "first_pin", "second_pin", "expected"), + [ + ("2026-10-03", "2026-10-02", PIN, COMMIT, 0), + ("2026-10-02", "2026-10-03", COMMIT, PIN, 1), + ("2026-10-02", "2026-10-02", PIN, COMMIT, 1), + ("", "2026-10-02", COMMIT, PIN, 0), + (None, None, PIN, COMMIT, 1), + ("2026-10-03", 123, PIN, COMMIT, 1), + ("", "", PIN, PIN, 1), + ], +) +def test_registered_orders_frozen_rows( + tmp_path: Path, + registration_store: Mock, + first_created: object, + second_created: object, + first_pin: str, + second_pin: str, + expected: int, +) -> None: + manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64)) + ex.write_units(manifest.units_file, [unit("u", "g", 1)]) + frozen = ex.register(registration_store, manifest, commit_of=committed) + rows = [dict(frozen, pin=pin, created=created) for pin, created in ((first_pin, first_created), (second_pin, second_created))] + for row in rows: + if row["created"] is None: + del row["created"] + # A run row and another experiment must not participate in registration selection. + ignored = [dict(frozen, task=ex.RUN_TASK), dict(frozen, repo="experiments/other")] + registration_store.rows.side_effect = lambda *, repo, kind: [Record(r, "memory", None) for r in rows + ignored if r["repo"] == repo] + assert ex.registered(registration_store, manifest.id) is rows[expected] + + def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None: store = FileStore(tmp_path / "memory") store.put_rows(example_rows()) diff --git a/tests/test_learn_fill.py b/tests/test_learn_fill.py new file mode 100644 index 0000000..139fdd7 --- /dev/null +++ b/tests/test_learn_fill.py @@ -0,0 +1,291 @@ +"""Rule cache filling uses scripted clients and an isolated local store; no provider is contacted.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +from learn_fixtures import ScriptedChat, default_answer +from test_learn_experiments import CODE, EXAMPLES, PRICES, excerpt_text, paid +from test_learn_rules import prepared, write_rule_manifest + +from openultrasast.cli import main +from openultrasast.learn import experiments as ex +from openultrasast.learn.fill import fill_rule +from openultrasast.learn.program import Candidate, Program, Prompt +from openultrasast.learn.retrieve import Target +from openultrasast.plane.memory import FileStore + + +def snapshot(root: Path) -> dict[str, bytes]: + return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} + + +def seed_one(store: FileStore, manifest: ex.Manifest, unit: ex.Unit) -> int: + chat, caller = paid() + caller.store = store + example = next(e for e in EXAMPLES if e.id == unit.unit) + folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]} + program = Program(ex.arm_spec(manifest, manifest.arms["A"], unit.family, store), EXAMPLES, excerpt_text) + program.decide(Candidate(Target.of(example), CODE[example.excerpt_sha], "python"), folds[unit.fold], caller, seed=0) + return len(chat.calls) + + +def total(report: dict[str, Any], key: str) -> int: + return sum(f[key] for f in report["families"].values()) + + +def test_dry_run_makes_zero_calls_and_writes_nothing(tmp_path: Path) -> None: + store, manifest, units = prepared(tmp_path) + cached = seed_one(store, manifest, units[0]) + chat = ScriptedChat() + before = snapshot(tmp_path) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, dry_run=True) + assert report["status"] == "dry_run" and report["client_calls"] == 0 and report["usd"] == 0 + assert not chat.calls and snapshot(tmp_path) == before + assert total(report, "units") == len(units) + assert total(report, "requests") == 2 * len(units) + assert total(report, "cached") == cached + assert total(report, "missing") == 2 * len(units) - cached + assert total(report, "filled") == 0 + assert all(f["estimated_usd"] > 0 for f in report["families"].values() if f["missing"]) + + +def test_fill_only_missing_requests_then_replay_is_complete(tmp_path: Path) -> None: + store, manifest, units = prepared(tmp_path) + cached = seed_one(store, manifest, units[0]) + chat = ScriptedChat() + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0) + assert report["status"] == "done" and total(report, "still_missing") == 0 + assert total(report, "filled") == report["client_calls"] == len(chat.calls) == 2 * len(units) - cached + assert 0 < report["usd"] < 1.0 + before = snapshot(tmp_path) + again = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0) + assert again["client_calls"] == 0 and total(again, "filled") == 0 and snapshot(tmp_path) == before + caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store) + replay = ex.run_rule(store, manifest, EXAMPLES, caller, meter, excerpt_text) + assert replay.decided == {"score": len(units)} and replay.skipped["replay_miss"] == 0 + assert meter.calls == 0 and meter.usd == 0 + + +@pytest.mark.parametrize("explicit_miss", [False, True]) +def test_dry_run_uses_mean_billed_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, explicit_miss: bool) -> None: + store, manifest, units = prepared(tmp_path) + assert seed_one(store, manifest, units[0]) == 2 + paths = sorted((tmp_path / "memory" / "responses").rglob("*")) + responses = [p for p in paths if p.is_file()] + assert len(responses) == 2 + for i, path in enumerate(responses, 1): + entry = json.loads(path.read_bytes()) + entry["usage"] = {"prompt_cache_hit_tokens": 9000 * i, "completion_tokens": 100 * i} + entry["usage"]["prompt_cache_miss_tokens" if explicit_miss else "prompt_tokens"] = (1000 if explicit_miss else 10000) * i + path.write_text(json.dumps(entry)) + monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1_000_000)) + before = snapshot(tmp_path) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True) + family = report["families"][units[0].family] + mean_cost = 1.5 * (9000 * 0.014 + 1000 * 0.44 + 100 * 1.32) / 1_000_000 + assert family["missing"] > 0 and family["estimate_method"] == "measured" + assert family["estimated_usd"] == pytest.approx(family["missing"] * mean_cost) + other = next(f for name, f in report["families"].items() if name != units[0].family) + assert other["estimate_method"] == "estimated" + assert report["client_calls"] == 0 and snapshot(tmp_path) == before + + +@pytest.mark.parametrize("usage", [None, {}]) +def test_dry_run_falls_back_without_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, usage: dict[str, int] | None) -> None: + store, manifest, units = prepared(tmp_path) + assert seed_one(store, manifest, units[0]) == 2 + for path in (tmp_path / "memory" / "responses").rglob("*"): + if path.is_file(): + entry = json.loads(path.read_bytes()) + entry.pop("usage") + if usage is not None: + entry["usage"] = usage + path.write_text(json.dumps(entry)) + monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1000)) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True) + for family in report["families"].values(): + assert family["estimate_method"] == "estimated" + assert family["estimated_output_tokens"] == 300 + assert family["estimated_usd"] == pytest.approx(family["missing"] * (1000 * 0.44 + 300 * 1.32) / 1_000_000) + + +class FixedCostChat(ScriptedChat): + def complete(self, **kwargs: Any) -> Any: + response = super().complete(**kwargs) + self.usage[-1] = {"prompt_tokens": 0, "completion_tokens": 6_000} + return response + + +def test_hard_ceiling_stops_before_a_non_multiple_budget_is_exceeded(tmp_path: Path) -> None: + store, manifest, _ = prepared(tmp_path) + chat = FixedCostChat() + prices = {"cache_hit_per_m": 0.0, "input_per_m": 0.0, "output_per_m": 1.0} + report = fill_rule(store, manifest, EXAMPLES, prices, excerpt_text, client=chat, budget_usd=0.025) + assert report["status"] == "unfinished" + assert 0 < report["usd"] <= 0.025 + assert report["usd"] == pytest.approx(len(chat.calls) * 0.006) + assert report["client_calls"] == total(report, "filled") == len(chat.calls) + assert total(report, "still_missing") > 0 + assert total(report, "filled") + total(report, "still_missing") == total(report, "missing") + + +def test_smoke_failure_stops_before_any_fill_and_redacts_provider_error(tmp_path: Path) -> None: + store, manifest, _ = prepared(tmp_path) + + def fail(*args: Any) -> str: + raise RuntimeError("provider unavailable SECRET_DO_NOT_PRINT") + + chat = ScriptedChat(fail) + before = snapshot(tmp_path) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0) + assert report["status"] == "failed" and report["client_calls"] == len(chat.calls) == 1 + assert total(report, "filled") == 0 and total(report, "still_missing") == total(report, "missing") + assert snapshot(tmp_path) == before + assert "SECRET_DO_NOT_PRINT" not in json.dumps(report) + + +def test_pending_units_refuse_before_calls_or_writes(tmp_path: Path) -> None: + store, original, _ = prepared(tmp_path) + path = write_rule_manifest(tmp_path, original.programs["injection"], "exp-902-pending") + manifest = ex.load_manifest(path) + ex.register(store, manifest, commit_of=lambda _: "c" * 40) + chat = ScriptedChat() + before = snapshot(tmp_path) + with pytest.raises(ex.ExperimentError, match="pending|frozen|units"): + fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0) + assert not chat.calls and snapshot(tmp_path) == before + + +@pytest.mark.parametrize("budget", [None, -1.0, float("nan"), float("inf")]) +def test_paid_fill_requires_a_finite_nonnegative_budget(tmp_path: Path, budget: float | None) -> None: + store, manifest, _ = prepared(tmp_path) + chat = ScriptedChat() + before = snapshot(tmp_path) + with pytest.raises(ex.ExperimentError, match="budget-usd"): + fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=budget) + assert not chat.calls and snapshot(tmp_path) == before + + +def test_zero_budget_stops_before_smoke(tmp_path: Path) -> None: + store, manifest, _ = prepared(tmp_path) + chat = ScriptedChat() + before = snapshot(tmp_path) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=0.0) + assert report["status"] == "unfinished" and report["client_calls"] == 0 and report["usd"] == 0 + assert not chat.calls and snapshot(tmp_path) == before + + +def test_parse_retry_requests_are_cached_for_identical_replay(tmp_path: Path) -> None: + store, manifest, units = prepared(tmp_path) + answers = 0 + + def first_invalid(messages: Any, temperature: Any) -> str: + nonlocal answers + answers += 1 + return "invalid json" if answers == 1 else default_answer(messages, temperature) + + chat = ScriptedChat(first_invalid) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, client=chat, budget_usd=1.0) + assert report["status"] == "done" and total(report, "still_missing") == 0 + assert len(chat.calls) == 2 * len(units) + 1 + caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store) + replay = ex.run_rule(store, manifest, EXAMPLES, caller, meter, excerpt_text) + assert replay.decided == {"score": len(units)} and replay.skipped["replay_miss"] == 0 + assert meter.calls == 0 + + +def test_cli_dry_run_needs_no_endpoint_and_does_not_write_out( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + import openultrasast.learn.examples as examples + import openultrasast.model.endpoint as endpoint + + _, manifest, _ = prepared(tmp_path) + real_get_blob = FileStore.get_blob + + def get_blob(self: FileStore, kind: str, sha: str, **kwargs: Any) -> bytes | None: + if kind == "excerpts": + text = CODE.get(sha) + return text.encode() if text is not None else None + return real_get_blob(self, kind, sha, **kwargs) + + monkeypatch.setattr(FileStore, "get_blob", get_blob) + monkeypatch.setattr(examples, "load_examples", lambda *_: EXAMPLES) + monkeypatch.setattr(endpoint, "resolve_chat_endpoint", lambda *_: pytest.fail("dry run resolved an endpoint")) + out = tmp_path / "report.json" + before = snapshot(tmp_path) + code = main( + ["learn", "experiment", "fill", str(manifest.path), "--dry-run", "--memory", f"file://{tmp_path / 'memory'}", "--out", str(out)] + ) + assert code == 0 and not out.exists() and snapshot(tmp_path) == before + report = json.loads(capsys.readouterr().out) + assert report["client_calls"] == 0 and report["status"] == "dry_run" + assert all(f["estimate_method"] == "estimated" for f in report["families"].values()) + + +def test_growing_call_costs_stay_inside_the_initial_reserve(tmp_path: Path) -> None: + store, manifest, _ = prepared(tmp_path) + + class GrowingCostChat(ScriptedChat): + def complete(self, **kwargs: Any) -> Any: + response = super().complete(**kwargs) + self.usage[-1] = {"prompt_tokens": 0, "completion_tokens": min(len(self.calls), 9) * 1_000} + return response + + chat = GrowingCostChat() + prices = {"cache_hit_per_m": 0.0, "input_per_m": 0.0, "output_per_m": 1.0} + report = fill_rule(store, manifest, EXAMPLES, prices, excerpt_text, client=chat, budget_usd=0.025) + assert report["status"] == "unfinished" and 0 < report["usd"] <= 0.025 + assert report["usd"] == pytest.approx(sum(int(row["completion_tokens"]) for row in chat.usage) / 1_000_000) + assert total(report, "filled") == len(chat.calls) and total(report, "still_missing") > 0 + + +def test_dry_run_prices_prompt_tokens_with_empty_cache(tmp_path: Path) -> None: + from openultrasast.learn.fill import DEFAULT_OUTPUT_TOKENS + + store, manifest, units = prepared(tmp_path) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True) + folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]} + index = {e.id: e for e in EXAMPLES} + expected = dict.fromkeys(manifest.families, 0.0) + for unit in units: + example = index[unit.unit] + program = Program(ex.arm_spec(manifest, manifest.arms["A"], unit.family, store), EXAMPLES, excerpt_text) + prompt, _ = program.prepare(Candidate(Target.of(example), CODE[example.excerpt_sha], "python"), folds[unit.fold], seed=0) + output_tokens = DEFAULT_OUTPUT_TOKENS + expected[unit.family] += ( + program.spec.k * (prompt.estimated_tokens * PRICES["input_per_m"] + output_tokens * PRICES["output_per_m"]) / 1_000_000 + ) + for family, amount in expected.items(): + assert report["families"][family]["estimated_usd"] == pytest.approx(amount) + + +def test_bounded_provider_caps_output_and_disables_unmetered_retries(monkeypatch: pytest.MonkeyPatch) -> None: + from openultrasast.model.endpoint import DeepSeekChatClient + from openultrasast.provider.openrouter import OpenRouterChatClient + + raw_calls: list[dict[str, Any]] = [] + + def raw(self: OpenRouterChatClient, **kwargs: Any) -> dict[str, object]: + raw_calls.append(kwargs) + assert self.max_attempts == 1 + return {"choices": [{"message": {"role": "assistant", "content": ""}}], "usage": {"prompt_tokens": 20, "completion_tokens": 3}} + + monkeypatch.setattr(OpenRouterChatClient, "complete_chat_raw", raw) + transport = OpenRouterChatClient(api_key="test-placeholder", max_attempts=3) + original = DeepSeekChatClient(transport, disable_thinking=True) # the default: config.models.thinking is False + bounded = original.bounded(300) + response = bounded.complete(model="deepseek-flash", messages=[{"role": "user", "content": "JSON"}], tools=[], json_object=True) + assert response.content == "" and len(raw_calls) == 1 + assert raw_calls[0]["extra_body"]["max_tokens"] == 300 + # The bounded copy keeps the configured client's thinking setting, so a fill answers as the run would. + assert raw_calls[0]["extra_body"]["thinking"] == {"type": "disabled"} + thinking_on = DeepSeekChatClient(transport, disable_thinking=False).bounded(300) + thinking_on.complete(model="deepseek-flash", messages=[{"role": "user", "content": "JSON"}], tools=[], json_object=True) + assert "thinking" not in raw_calls[-1]["extra_body"] + assert bounded.usage == [{"prompt_tokens": 20, "completion_tokens": 3}] + assert original.usage == [] and transport.max_attempts == 3