From d9608b9a63fd99e4502cb19e128f797f5404214b Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:40:00 +0200 Subject: [PATCH] bench(learn): harvest the advisory-fix pairs The 97 advisory-fix pairs (194 sides) harvested for the decision engine: verify a/b + agree on the 34 untrusted_destination sides and model roles on every side as plane Runs on ax ($0.38 metered, task budgets $1.49, ceiling $5; DeepSeek balance 17.88 -> 17.73), engine features on the host (frozen 7b17eac, one container, --memory 3g: 54 ran, 58 asked no question, 82 no language model), feature records for all 194 sides, static memory rebuilt with the merged harvest units: 724 -> 904 examples. No unit was skipped for the 20 KB inline limit. Instrument fix: SideReader read only the first pair-kind source, so the first rebuild left all 194 advisory sides `unread` (0 examples). It now reads every pair-kind source; test added. Leak audit: the static profile flags only engine signals and facts.function_lines (withheld by v1); the plane profile newly flags verify.flag_b and ms.operation (plane-only, not in any v1 prompt). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .kiro/specs/learned-decision-engine/tasks.md | 4 + .../record.json | 778 ++++++++++++++++++ src/openultrasast/learn/examples.py | 6 +- tests/test_learn_examples.py | 31 + 4 files changed, 817 insertions(+), 2 deletions(-) create mode 100644 benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json diff --git a/.kiro/specs/learned-decision-engine/tasks.md b/.kiro/specs/learned-decision-engine/tasks.md index e268a3b..fa0c744 100644 --- a/.kiro/specs/learned-decision-engine/tasks.md +++ b/.kiro/specs/learned-decision-engine/tasks.md @@ -38,6 +38,10 @@ a model or embedding call. - Done 2026-10-01 (`benchmarks/measurements/2026-10-01-decision-engine-harvest/record.json`): verify $2.54 and roles $1.40 by attribution. The engine job finished the same day (682 of 686 pair sides; the 2 family-unknown pairs not run) and the feature records were rebuilt; dev-php pins have no engine run (`missing`). + - Advisory-fix pairs harvested 2026-10-02 (`benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json`): + 194 sides, verify a/b on the 34 untrusted_destination sides and model roles on all ($0.38 metered, ceiling $5), + engine on the host (54 ran, 58 asked no question, 82 no language model); static memory 724 -> 904 examples. The + leak audit flags nothing new in the static profile; the plane profile newly flags `verify.flag_b` and `ms.operation`. - _Requirements: 1.1_ - [ ] 6. Program, local memory and compilation (design sections 4-5) diff --git a/benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json b/benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json new file mode 100644 index 0000000..bb72dde --- /dev/null +++ b/benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json @@ -0,0 +1,778 @@ +{ + "created": "2026-10-02", + "embeddings": { + "cached": 616, + "calls": 6, + "embedded": 172, + "model": "openai/text-embedding-3-small", + "requested": 788, + "tokens": 43397, + "usd": 0.0008679400000000001 + }, + "engine": { + "by_side_state": { + "fixed:failed": 30, + "fixed:none": 41, + "fixed:ran": 26, + "vuln:failed": 28, + "vuln:none": 41, + "vuln:ran": 28 + }, + "frozen_source_commit": "7b17eacd44fa22ba56d96da3af1491cef2331987", + "image": [ + "sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb" + ], + "not_ran_reasons": { + "failed (read its input, asked no question)": 58, + "no engine model for csharp": 18, + "no engine model for go": 22, + "no engine model for ruby": 22, + "no engine model for typescript": 20 + }, + "note": "none = no engine model for the language; failed = read its input and asked no question (never zero findings)", + "ran_with_findings": 3, + "seconds_total": 7400, + "sides": 194, + "state": { + "failed": 58, + "none": 82, + "ran": 54 + } + }, + "features": { + "absent_on_side": {}, + "dropped_stale_rows": 0, + "instrument_states_plane": { + "access_control:agree": { + "none": 40 + }, + "access_control:delta": { + "not_applicable": 40 + }, + "access_control:engine": { + "failed": 10, + "none": 20, + "ran": 10 + }, + "access_control:entry_points": { + "not_applicable": 40 + }, + "access_control:facts": { + "not_applicable": 40 + }, + "access_control:language": { + "ran": 40 + }, + "access_control:model_sinks": { + "ran": 40 + }, + "access_control:quick": { + "none": 16, + "ran": 24 + }, + "access_control:roles": { + "failed": 4, + "none": 4, + "ran": 32 + }, + "access_control:source": { + "failed": 4, + "none": 4, + "ran": 32 + }, + "access_control:verify": { + "none": 40 + }, + "deserialization:agree": { + "none": 42 + }, + "deserialization:delta": { + "not_applicable": 42 + }, + "deserialization:engine": { + "failed": 15, + "none": 14, + "ran": 13 + }, + "deserialization:entry_points": { + "not_applicable": 42 + }, + "deserialization:facts": { + "not_applicable": 42 + }, + "deserialization:language": { + "ran": 42 + }, + "deserialization:model_sinks": { + "ran": 42 + }, + "deserialization:quick": { + "none": 12, + "ran": 30 + }, + "deserialization:roles": { + "none": 6, + "ran": 36 + }, + "deserialization:source": { + "none": 6, + "ran": 36 + }, + "deserialization:verify": { + "none": 42 + }, + "output_encoding:agree": { + "none": 42 + }, + "output_encoding:delta": { + "not_applicable": 42 + }, + "output_encoding:engine": { + "failed": 15, + "none": 20, + "ran": 7 + }, + "output_encoding:entry_points": { + "not_applicable": 42 + }, + "output_encoding:facts": { + "not_applicable": 42 + }, + "output_encoding:language": { + "ran": 42 + }, + "output_encoding:model_sinks": { + "ran": 42 + }, + "output_encoding:quick": { + "none": 12, + "ran": 30 + }, + "output_encoding:roles": { + "failed": 2, + "none": 4, + "ran": 36 + }, + "output_encoding:source": { + "failed": 2, + "none": 4, + "ran": 36 + }, + "output_encoding:verify": { + "none": 42 + }, + "path:agree": { + "none": 36 + }, + "path:delta": { + "not_applicable": 36 + }, + "path:engine": { + "failed": 6, + "none": 16, + "ran": 14 + }, + "path:entry_points": { + "not_applicable": 36 + }, + "path:facts": { + "not_applicable": 36 + }, + "path:language": { + "ran": 36 + }, + "path:model_sinks": { + "ran": 36 + }, + "path:quick": { + "none": 14, + "ran": 22 + }, + "path:roles": { + "none": 2, + "ran": 34 + }, + "path:source": { + "none": 2, + "ran": 34 + }, + "path:verify": { + "none": 36 + }, + "untrusted_destination:agree": { + "ran": 34 + }, + "untrusted_destination:delta": { + "not_applicable": 34 + }, + "untrusted_destination:engine": { + "failed": 12, + "none": 12, + "ran": 10 + }, + "untrusted_destination:entry_points": { + "not_applicable": 34 + }, + "untrusted_destination:facts": { + "not_applicable": 34 + }, + "untrusted_destination:language": { + "ran": 34 + }, + "untrusted_destination:model_sinks": { + "ran": 34 + }, + "untrusted_destination:quick": { + "none": 8, + "ran": 26 + }, + "untrusted_destination:roles": { + "failed": 2, + "none": 2, + "ran": 30 + }, + "untrusted_destination:source": { + "failed": 2, + "none": 2, + "ran": 30 + }, + "untrusted_destination:verify": { + "ran": 34 + } + }, + "records": { + "plane:access_control": 40, + "plane:deserialization": 42, + "plane:output_encoding": 42, + "plane:path": 36, + "plane:untrusted_destination": 34, + "static:access_control": 40, + "static:deserialization": 42, + "static:output_encoding": 42, + "static:path": 36, + "static:untrusted_destination": 34 + }, + "skipped_label_rows": {} + }, + "host_peaks": { + "disk_free_mb_min": 2630, + "mem_available_mb_min": 2341 + }, + "labels": { + "advisory_pin_rows": 194, + "note": "regenerated 2026-10-02; identical to the 2026-10-01 snapshot apart from the `created` field", + "rows": 6589, + "same_counts_as": "2026-10-01-decision-engine-labels-advisory-fixes/labels.json" + }, + "leak_audit": { + "command": "ousast learn audit-leaks --profile {plane,static} --units (threshold 0.20, min 10 pairs)", + "pairs": { + "access_control": 20, + "all": 376, + "config_secrets": 12, + "deserialization": 21, + "injection": 43, + "memory": 171, + "output_encoding": 61, + "path": 19, + "unknown": 2, + "untrusted_destination": 27 + }, + "plane": { + "flagged": [ + { + "correct": 10, + "direction": "higher on fixed", + "pairs": 12, + "scope": "config_secrets", + "separation": 0.75, + "signal": "facts.function_lines", + "wrong": 1 + }, + { + "correct": 16, + "direction": "higher on fixed", + "pairs": 19, + "scope": "path", + "separation": 0.7368, + "signal": "facts.function_lines", + "wrong": 2 + }, + { + "correct": 8, + "direction": "higher on fixed", + "pairs": 12, + "scope": "config_secrets", + "separation": 0.6667, + "signal": "verify.turns_mean", + "wrong": 0 + }, + { + "correct": 13, + "direction": "higher on fixed", + "pairs": 20, + "scope": "access_control", + "separation": 0.6, + "signal": "facts.function_lines", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.5556, + "signal": "agree.final", + "wrong": 0 + }, + { + "correct": 18, + "direction": "higher on fixed", + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.5556, + "signal": "facts.function_lines", + "wrong": 3 + }, + { + "correct": 15, + "direction": "higher on vulnerable", + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.5556, + "signal": "verify.votes", + "wrong": 0 + }, + { + "correct": 14, + "direction": "higher on vulnerable", + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.5185, + "signal": "verify.flag_a", + "wrong": 0 + }, + { + "correct": 15, + "direction": "higher on vulnerable", + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.4815, + "signal": "verify.site_offset", + "wrong": 2 + }, + { + "correct": 12, + "direction": "higher on vulnerable", + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.4074, + "signal": "verify.flag_b", + "wrong": 1 + }, + { + "correct": 6, + "direction": null, + "pairs": 20, + "scope": "access_control", + "separation": 0.3, + "signal": "ms.operation", + "wrong": 0 + }, + { + "correct": 9, + "direction": "higher on fixed", + "pairs": 21, + "scope": "deserialization", + "separation": 0.2857, + "signal": "facts.function_lines", + "wrong": 3 + }, + { + "correct": 18, + "direction": "higher on vulnerable", + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2623, + "signal": "eng.witness_steps_min", + "wrong": 2 + }, + { + "correct": 7, + "direction": null, + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.2593, + "signal": "ms.operation", + "wrong": 0 + }, + { + "correct": 21, + "direction": "higher on fixed", + "pairs": 43, + "scope": "injection", + "separation": 0.2558, + "signal": "facts.function_lines", + "wrong": 10 + }, + { + "correct": 3, + "direction": null, + "pairs": 12, + "scope": "config_secrets", + "separation": 0.25, + "signal": "agree.final", + "wrong": 0 + }, + { + "correct": 3, + "direction": "higher on vulnerable", + "pairs": 12, + "scope": "config_secrets", + "separation": 0.25, + "signal": "verify.flag_a", + "wrong": 0 + }, + { + "correct": 3, + "direction": "higher on vulnerable", + "pairs": 12, + "scope": "config_secrets", + "separation": 0.25, + "signal": "verify.site_offset", + "wrong": 0 + }, + { + "correct": 3, + "direction": "higher on vulnerable", + "pairs": 12, + "scope": "config_secrets", + "separation": 0.25, + "signal": "verify.votes", + "wrong": 0 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.rung_max", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.sink_kind", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.sink_origin", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.source_kind", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.source_origin", + "wrong": 1 + }, + { + "correct": 15, + "direction": "higher on vulnerable", + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2131, + "signal": "eng.findings", + "wrong": 2 + } + ], + "flagged_signals": [ + "agree.final", + "eng.findings", + "eng.rung_max", + "eng.sink_kind", + "eng.sink_origin", + "eng.source_kind", + "eng.source_origin", + "eng.witness_steps_min", + "facts.function_lines", + "ms.operation", + "verify.flag_a", + "verify.flag_b", + "verify.site_offset", + "verify.turns_mean", + "verify.votes" + ], + "new_signal_names_vs_2026_10_01": [ + "ms.operation", + "verify.flag_b" + ] + }, + "reading": "The static profile (the one slice 2 runs) flags only engine signals and facts.function_lines, both withheld by input profile v1. The plane profile newly flags verify.flag_b and ms.operation (plane-only instruments, never in a static/v1 prompt); verify flags point to the vulnerable side, as a real detector would, but whether they track the label must be decided before any plane-profile program runs.", + "static": { + "flagged": [ + { + "correct": 10, + "direction": "higher on fixed", + "pairs": 12, + "scope": "config_secrets", + "separation": 0.75, + "signal": "facts.function_lines", + "wrong": 1 + }, + { + "correct": 16, + "direction": "higher on fixed", + "pairs": 19, + "scope": "path", + "separation": 0.7368, + "signal": "facts.function_lines", + "wrong": 2 + }, + { + "correct": 13, + "direction": "higher on fixed", + "pairs": 20, + "scope": "access_control", + "separation": 0.6, + "signal": "facts.function_lines", + "wrong": 1 + }, + { + "correct": 18, + "direction": "higher on fixed", + "pairs": 27, + "scope": "untrusted_destination", + "separation": 0.5556, + "signal": "facts.function_lines", + "wrong": 3 + }, + { + "correct": 9, + "direction": "higher on fixed", + "pairs": 21, + "scope": "deserialization", + "separation": 0.2857, + "signal": "facts.function_lines", + "wrong": 3 + }, + { + "correct": 18, + "direction": "higher on vulnerable", + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2623, + "signal": "eng.witness_steps_min", + "wrong": 2 + }, + { + "correct": 21, + "direction": "higher on fixed", + "pairs": 43, + "scope": "injection", + "separation": 0.2558, + "signal": "facts.function_lines", + "wrong": 10 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.rung_max", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.sink_kind", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.sink_origin", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.source_kind", + "wrong": 1 + }, + { + "correct": 15, + "direction": null, + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2295, + "signal": "eng.source_origin", + "wrong": 1 + }, + { + "correct": 15, + "direction": "higher on vulnerable", + "pairs": 61, + "scope": "output_encoding", + "separation": 0.2131, + "signal": "eng.findings", + "wrong": 2 + } + ], + "flagged_signals": [ + "eng.findings", + "eng.rung_max", + "eng.sink_kind", + "eng.sink_origin", + "eng.source_kind", + "eng.source_origin", + "eng.witness_steps_min", + "facts.function_lines" + ], + "outside_v1_withheld": [] + } + }, + "memory_build": { + "advisory_sides_not_examples": { + "absent_fixed_side": 7, + "no_excerpt": 7 + }, + "by_source": { + "adjudications": { + "absent_fixed_side": 0, + "examples": 34, + "labels": 43, + "no_excerpt": 9, + "no_record": 0, + "recorded": 43, + "unread": 0 + }, + "dev-php": { + "absent_fixed_side": 0, + "examples": 3, + "labels": 3, + "no_excerpt": 0, + "no_record": 0, + "recorded": 3, + "unread": 0 + }, + "pairs": { + "absent_fixed_side": 7, + "examples": 740, + "labels": 1544, + "no_excerpt": 7, + "no_record": 790, + "recorded": 754, + "unread": 0 + }, + "population-v1": { + "absent_fixed_side": 0, + "examples": 85, + "labels": 170, + "no_excerpt": 0, + "no_record": 85, + "recorded": 85, + "unread": 0 + }, + "population-v2": { + "absent_fixed_side": 0, + "examples": 42, + "labels": 86, + "no_excerpt": 1, + "no_record": 43, + "recorded": 43, + "unread": 0 + } + }, + "examples": 904, + "examples_added": 180, + "examples_before": 724, + "failing_reason": "adjudications: 9 of 43 anonymous functions, unchanged since 2026-10-01 (exit 2 recorded, rows kept)", + "failing_sources": [ + "adjudications" + ], + "instrument_fix": "SideReader read only the first pair-kind source, so all 194 advisory sides were `unread` (0 examples) on the first rebuild; fixed in learn/examples.py with a test; the rebuild then read every side", + "profile": "static" + }, + "note": "Decision-engine harvest of the 97 advisory-fix pairs (benchmarks/pairs/advisory-fixes, 194 sides), COUNTS ONLY. Plane Runs hadv-verify (verify a/b + agree; verify families only: untrusted_destination) and hadv-roles (model roles, every side) on ax, generated by `ousast plane harvest --catalog benchmarks/pairs/advisory-fixes/catalog.toml --ceiling 2` from the label rows of those pairs; host engine features (benchmarks/learn/engine_pairs.py, frozen source, one container, --memory 3g); feature records (benchmarks/learn/build_features.py); static memory rebuilt with --units (old + new harvest units merged). Unit lists, label rows and excerpts name functions and stay local.", + "roles": { + "files_classified": 194, + "role_rows": 660, + "sides": 194, + "skipped_units": { + "oversized_over_20KB_inline": 0 + }, + "tasks_done": 20 + }, + "runner_image": "localhost:5001/ousast-runner@sha256:412db0a1504af8f1b23a395af8d1dcbd152c1bb6d16594f2b733067163a31884 (not rebuilt: no change under plane/tasks since it was built)", + "spend": { + "ceiling_usd": 5.0, + "deepseek_balance_usd": { + "after_features": 17.73, + "after_runs": 17.75, + "before": 17.88 + }, + "embeddings_openrouter_usd": 0.000868, + "metered_usd": { + "pilot_included": true, + "roles": 0.1285, + "total": 0.3802, + "verify": 0.2517 + }, + "smoke": "plain JSON + one tool call through verify.build_client before any Run (one reply carried a stray `<|OPENAI|>` token in JSON mode on a trivial prompt; 0/16 on a classifier-shaped probe)", + "task_budget_sum_usd": { + "roles": 0.669, + "verify": 0.816 + } + }, + "verify": { + "flagged_by_pass_units": { + "fixed:none": 16, + "fixed:one": 1, + "vulnerable:both": 10, + "vulnerable:none": 4, + "vulnerable:one": 3 + }, + "hunts_per_pass": 34, + "skipped_units": { + "oversized_over_20KB_inline": 0 + }, + "tasks_done": 102, + "units": 34, + "usage": { + "completion_tokens": 66734, + "prompt_cache_hit_tokens": 444926, + "prompt_tokens": 802586 + } + } +} diff --git a/src/openultrasast/learn/examples.py b/src/openultrasast/learn/examples.py index c24842e..4964d85 100644 --- a/src/openultrasast/learn/examples.py +++ b/src/openultrasast/learn/examples.py @@ -265,8 +265,10 @@ class SideReader: if self._pairs is None: from ..pairs import load_pair_catalog - source = next(s for s in self.sources.sources if s.kind == "pairs") - self._pairs = {case.name: case for case in load_pair_catalog(self.root / source.files[0])} + # every pair-kind source (the corpus catalog and e.g. advisory-fixes): labels name both as source "pairs" + self._pairs = {} + for source in (s for s in self.sources.sources if s.kind == "pairs"): + self._pairs.update({case.name: case for case in load_pair_catalog(self.root / source.files[0])}) return self._pairs def license(self, label: Mapping[str, Any]) -> str: diff --git a/tests/test_learn_examples.py b/tests/test_learn_examples.py index e60fe6a..b18b5a2 100644 --- a/tests/test_learn_examples.py +++ b/tests/test_learn_examples.py @@ -171,3 +171,34 @@ def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> Non assert build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES).summary()["examples"] == 0 build = build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES, pins=pins) assert build.summary()["examples"] == 1 and build.rows[0]["pin"] == unit_pin + + +def test_side_reader_reads_every_pair_kind_source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Labels from the corpus catalog and from a second pair-kind source (advisory-fixes) are both source ``pairs``; + the reader must find either side, not only the first catalog's (the 2026-10-02 advisory rebuild read 0 of 194).""" + from types import SimpleNamespace + + import openultrasast.pairs as pairs + from openultrasast.learn.examples import SideReader + + files = {} + for name in ("corpus-case", "advisory-case"): + files[name] = tmp_path / f"{name}.py" + files[name].write_text("def handler(request):\n return 1\n") + catalogs = {"corpus.toml": "corpus-case", "advisory.toml": "advisory-case"} + + def fake(path: Path) -> tuple[Any, ...]: + name = catalogs[Path(path).name] + return (SimpleNamespace(name=name, vuln_file=files[name], fixed_file=files[name], license="MIT"),) + + monkeypatch.setattr(pairs, "load_pair_catalog", fake) + sources = Sources( + (Source("pairs", "pairs", ("corpus.toml",), (), "2026-09-30", "development"), + Source("advisory-fixes", "pairs", ("advisory.toml",), (), "2026-10-01", "development")), + frozenset(), {}, Path("sources.toml"), + ) # fmt: skip + reader = SideReader(tmp_path, tmp_path, sources, []) + for name in catalogs.values(): + lab = label(source="pairs", source_ref=name, pin="") + assert reader(lab, "vulnerable") == ["def handler(request):", " return 1"] + assert reader.license(lab) == "MIT"