mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
bench(learn): harvest the advisory-fix pairs
The 97 advisory-fix pairs (194 sides) harvested for the decision engine: verify a/b + agree on the 34
untrusted_destination sides and model roles on every side as plane Runs on ax ($0.38 metered, task
budgets $1.49, ceiling $5; DeepSeek balance 17.88 -> 17.73), engine features on the host (frozen
7b17eac, one container, --memory 3g: 54 ran, 58 asked no question, 82 no language model), feature
records for all 194 sides, static memory rebuilt with the merged harvest units: 724 -> 904 examples.
No unit was skipped for the 20 KB inline limit.
Instrument fix: SideReader read only the first pair-kind source, so the first rebuild left all 194
advisory sides `unread` (0 examples). It now reads every pair-kind source; test added.
Leak audit: the static profile flags only engine signals and facts.function_lines (withheld by v1);
the plane profile newly flags verify.flag_b and ms.operation (plane-only, not in any v1 prompt).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7b17eacd44
commit
d9608b9a63
@@ -38,6 +38,10 @@ a model or embedding call.
|
||||
- Done 2026-10-01 (`benchmarks/measurements/2026-10-01-decision-engine-harvest/record.json`): verify $2.54 and roles
|
||||
$1.40 by attribution. The engine job finished the same day (682 of 686 pair sides; the 2 family-unknown pairs
|
||||
not run) and the feature records were rebuilt; dev-php pins have no engine run (`missing`).
|
||||
- Advisory-fix pairs harvested 2026-10-02 (`benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json`):
|
||||
194 sides, verify a/b on the 34 untrusted_destination sides and model roles on all ($0.38 metered, ceiling $5),
|
||||
engine on the host (54 ran, 58 asked no question, 82 no language model); static memory 724 -> 904 examples. The
|
||||
leak audit flags nothing new in the static profile; the plane profile newly flags `verify.flag_b` and `ms.operation`.
|
||||
- _Requirements: 1.1_
|
||||
|
||||
- [ ] 6. Program, local memory and compilation (design sections 4-5)
|
||||
|
||||
@@ -0,0 +1,778 @@
|
||||
{
|
||||
"created": "2026-10-02",
|
||||
"embeddings": {
|
||||
"cached": 616,
|
||||
"calls": 6,
|
||||
"embedded": 172,
|
||||
"model": "openai/text-embedding-3-small",
|
||||
"requested": 788,
|
||||
"tokens": 43397,
|
||||
"usd": 0.0008679400000000001
|
||||
},
|
||||
"engine": {
|
||||
"by_side_state": {
|
||||
"fixed:failed": 30,
|
||||
"fixed:none": 41,
|
||||
"fixed:ran": 26,
|
||||
"vuln:failed": 28,
|
||||
"vuln:none": 41,
|
||||
"vuln:ran": 28
|
||||
},
|
||||
"frozen_source_commit": "7b17eacd44fa22ba56d96da3af1491cef2331987",
|
||||
"image": [
|
||||
"sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb"
|
||||
],
|
||||
"not_ran_reasons": {
|
||||
"failed (read its input, asked no question)": 58,
|
||||
"no engine model for csharp": 18,
|
||||
"no engine model for go": 22,
|
||||
"no engine model for ruby": 22,
|
||||
"no engine model for typescript": 20
|
||||
},
|
||||
"note": "none = no engine model for the language; failed = read its input and asked no question (never zero findings)",
|
||||
"ran_with_findings": 3,
|
||||
"seconds_total": 7400,
|
||||
"sides": 194,
|
||||
"state": {
|
||||
"failed": 58,
|
||||
"none": 82,
|
||||
"ran": 54
|
||||
}
|
||||
},
|
||||
"features": {
|
||||
"absent_on_side": {},
|
||||
"dropped_stale_rows": 0,
|
||||
"instrument_states_plane": {
|
||||
"access_control:agree": {
|
||||
"none": 40
|
||||
},
|
||||
"access_control:delta": {
|
||||
"not_applicable": 40
|
||||
},
|
||||
"access_control:engine": {
|
||||
"failed": 10,
|
||||
"none": 20,
|
||||
"ran": 10
|
||||
},
|
||||
"access_control:entry_points": {
|
||||
"not_applicable": 40
|
||||
},
|
||||
"access_control:facts": {
|
||||
"not_applicable": 40
|
||||
},
|
||||
"access_control:language": {
|
||||
"ran": 40
|
||||
},
|
||||
"access_control:model_sinks": {
|
||||
"ran": 40
|
||||
},
|
||||
"access_control:quick": {
|
||||
"none": 16,
|
||||
"ran": 24
|
||||
},
|
||||
"access_control:roles": {
|
||||
"failed": 4,
|
||||
"none": 4,
|
||||
"ran": 32
|
||||
},
|
||||
"access_control:source": {
|
||||
"failed": 4,
|
||||
"none": 4,
|
||||
"ran": 32
|
||||
},
|
||||
"access_control:verify": {
|
||||
"none": 40
|
||||
},
|
||||
"deserialization:agree": {
|
||||
"none": 42
|
||||
},
|
||||
"deserialization:delta": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"deserialization:engine": {
|
||||
"failed": 15,
|
||||
"none": 14,
|
||||
"ran": 13
|
||||
},
|
||||
"deserialization:entry_points": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"deserialization:facts": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"deserialization:language": {
|
||||
"ran": 42
|
||||
},
|
||||
"deserialization:model_sinks": {
|
||||
"ran": 42
|
||||
},
|
||||
"deserialization:quick": {
|
||||
"none": 12,
|
||||
"ran": 30
|
||||
},
|
||||
"deserialization:roles": {
|
||||
"none": 6,
|
||||
"ran": 36
|
||||
},
|
||||
"deserialization:source": {
|
||||
"none": 6,
|
||||
"ran": 36
|
||||
},
|
||||
"deserialization:verify": {
|
||||
"none": 42
|
||||
},
|
||||
"output_encoding:agree": {
|
||||
"none": 42
|
||||
},
|
||||
"output_encoding:delta": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"output_encoding:engine": {
|
||||
"failed": 15,
|
||||
"none": 20,
|
||||
"ran": 7
|
||||
},
|
||||
"output_encoding:entry_points": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"output_encoding:facts": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"output_encoding:language": {
|
||||
"ran": 42
|
||||
},
|
||||
"output_encoding:model_sinks": {
|
||||
"ran": 42
|
||||
},
|
||||
"output_encoding:quick": {
|
||||
"none": 12,
|
||||
"ran": 30
|
||||
},
|
||||
"output_encoding:roles": {
|
||||
"failed": 2,
|
||||
"none": 4,
|
||||
"ran": 36
|
||||
},
|
||||
"output_encoding:source": {
|
||||
"failed": 2,
|
||||
"none": 4,
|
||||
"ran": 36
|
||||
},
|
||||
"output_encoding:verify": {
|
||||
"none": 42
|
||||
},
|
||||
"path:agree": {
|
||||
"none": 36
|
||||
},
|
||||
"path:delta": {
|
||||
"not_applicable": 36
|
||||
},
|
||||
"path:engine": {
|
||||
"failed": 6,
|
||||
"none": 16,
|
||||
"ran": 14
|
||||
},
|
||||
"path:entry_points": {
|
||||
"not_applicable": 36
|
||||
},
|
||||
"path:facts": {
|
||||
"not_applicable": 36
|
||||
},
|
||||
"path:language": {
|
||||
"ran": 36
|
||||
},
|
||||
"path:model_sinks": {
|
||||
"ran": 36
|
||||
},
|
||||
"path:quick": {
|
||||
"none": 14,
|
||||
"ran": 22
|
||||
},
|
||||
"path:roles": {
|
||||
"none": 2,
|
||||
"ran": 34
|
||||
},
|
||||
"path:source": {
|
||||
"none": 2,
|
||||
"ran": 34
|
||||
},
|
||||
"path:verify": {
|
||||
"none": 36
|
||||
},
|
||||
"untrusted_destination:agree": {
|
||||
"ran": 34
|
||||
},
|
||||
"untrusted_destination:delta": {
|
||||
"not_applicable": 34
|
||||
},
|
||||
"untrusted_destination:engine": {
|
||||
"failed": 12,
|
||||
"none": 12,
|
||||
"ran": 10
|
||||
},
|
||||
"untrusted_destination:entry_points": {
|
||||
"not_applicable": 34
|
||||
},
|
||||
"untrusted_destination:facts": {
|
||||
"not_applicable": 34
|
||||
},
|
||||
"untrusted_destination:language": {
|
||||
"ran": 34
|
||||
},
|
||||
"untrusted_destination:model_sinks": {
|
||||
"ran": 34
|
||||
},
|
||||
"untrusted_destination:quick": {
|
||||
"none": 8,
|
||||
"ran": 26
|
||||
},
|
||||
"untrusted_destination:roles": {
|
||||
"failed": 2,
|
||||
"none": 2,
|
||||
"ran": 30
|
||||
},
|
||||
"untrusted_destination:source": {
|
||||
"failed": 2,
|
||||
"none": 2,
|
||||
"ran": 30
|
||||
},
|
||||
"untrusted_destination:verify": {
|
||||
"ran": 34
|
||||
}
|
||||
},
|
||||
"records": {
|
||||
"plane:access_control": 40,
|
||||
"plane:deserialization": 42,
|
||||
"plane:output_encoding": 42,
|
||||
"plane:path": 36,
|
||||
"plane:untrusted_destination": 34,
|
||||
"static:access_control": 40,
|
||||
"static:deserialization": 42,
|
||||
"static:output_encoding": 42,
|
||||
"static:path": 36,
|
||||
"static:untrusted_destination": 34
|
||||
},
|
||||
"skipped_label_rows": {}
|
||||
},
|
||||
"host_peaks": {
|
||||
"disk_free_mb_min": 2630,
|
||||
"mem_available_mb_min": 2341
|
||||
},
|
||||
"labels": {
|
||||
"advisory_pin_rows": 194,
|
||||
"note": "regenerated 2026-10-02; identical to the 2026-10-01 snapshot apart from the `created` field",
|
||||
"rows": 6589,
|
||||
"same_counts_as": "2026-10-01-decision-engine-labels-advisory-fixes/labels.json"
|
||||
},
|
||||
"leak_audit": {
|
||||
"command": "ousast learn audit-leaks --profile {plane,static} --units <merged units.json> (threshold 0.20, min 10 pairs)",
|
||||
"pairs": {
|
||||
"access_control": 20,
|
||||
"all": 376,
|
||||
"config_secrets": 12,
|
||||
"deserialization": 21,
|
||||
"injection": 43,
|
||||
"memory": 171,
|
||||
"output_encoding": 61,
|
||||
"path": 19,
|
||||
"unknown": 2,
|
||||
"untrusted_destination": 27
|
||||
},
|
||||
"plane": {
|
||||
"flagged": [
|
||||
{
|
||||
"correct": 10,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.75,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 16,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 19,
|
||||
"scope": "path",
|
||||
"separation": 0.7368,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 8,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.6667,
|
||||
"signal": "verify.turns_mean",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 13,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 20,
|
||||
"scope": "access_control",
|
||||
"separation": 0.6,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "agree.final",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "verify.votes",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 14,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5185,
|
||||
"signal": "verify.flag_a",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.4815,
|
||||
"signal": "verify.site_offset",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 12,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.4074,
|
||||
"signal": "verify.flag_b",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 6,
|
||||
"direction": null,
|
||||
"pairs": 20,
|
||||
"scope": "access_control",
|
||||
"separation": 0.3,
|
||||
"signal": "ms.operation",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 9,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 21,
|
||||
"scope": "deserialization",
|
||||
"separation": 0.2857,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2623,
|
||||
"signal": "eng.witness_steps_min",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 7,
|
||||
"direction": null,
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.2593,
|
||||
"signal": "ms.operation",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 21,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 43,
|
||||
"scope": "injection",
|
||||
"separation": 0.2558,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 10
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": null,
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "agree.final",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "verify.flag_a",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "verify.site_offset",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "verify.votes",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.rung_max",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2131,
|
||||
"signal": "eng.findings",
|
||||
"wrong": 2
|
||||
}
|
||||
],
|
||||
"flagged_signals": [
|
||||
"agree.final",
|
||||
"eng.findings",
|
||||
"eng.rung_max",
|
||||
"eng.sink_kind",
|
||||
"eng.sink_origin",
|
||||
"eng.source_kind",
|
||||
"eng.source_origin",
|
||||
"eng.witness_steps_min",
|
||||
"facts.function_lines",
|
||||
"ms.operation",
|
||||
"verify.flag_a",
|
||||
"verify.flag_b",
|
||||
"verify.site_offset",
|
||||
"verify.turns_mean",
|
||||
"verify.votes"
|
||||
],
|
||||
"new_signal_names_vs_2026_10_01": [
|
||||
"ms.operation",
|
||||
"verify.flag_b"
|
||||
]
|
||||
},
|
||||
"reading": "The static profile (the one slice 2 runs) flags only engine signals and facts.function_lines, both withheld by input profile v1. The plane profile newly flags verify.flag_b and ms.operation (plane-only instruments, never in a static/v1 prompt); verify flags point to the vulnerable side, as a real detector would, but whether they track the label must be decided before any plane-profile program runs.",
|
||||
"static": {
|
||||
"flagged": [
|
||||
{
|
||||
"correct": 10,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.75,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 16,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 19,
|
||||
"scope": "path",
|
||||
"separation": 0.7368,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 13,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 20,
|
||||
"scope": "access_control",
|
||||
"separation": 0.6,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 9,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 21,
|
||||
"scope": "deserialization",
|
||||
"separation": 0.2857,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2623,
|
||||
"signal": "eng.witness_steps_min",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 21,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 43,
|
||||
"scope": "injection",
|
||||
"separation": 0.2558,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 10
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.rung_max",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2131,
|
||||
"signal": "eng.findings",
|
||||
"wrong": 2
|
||||
}
|
||||
],
|
||||
"flagged_signals": [
|
||||
"eng.findings",
|
||||
"eng.rung_max",
|
||||
"eng.sink_kind",
|
||||
"eng.sink_origin",
|
||||
"eng.source_kind",
|
||||
"eng.source_origin",
|
||||
"eng.witness_steps_min",
|
||||
"facts.function_lines"
|
||||
],
|
||||
"outside_v1_withheld": []
|
||||
}
|
||||
},
|
||||
"memory_build": {
|
||||
"advisory_sides_not_examples": {
|
||||
"absent_fixed_side": 7,
|
||||
"no_excerpt": 7
|
||||
},
|
||||
"by_source": {
|
||||
"adjudications": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 34,
|
||||
"labels": 43,
|
||||
"no_excerpt": 9,
|
||||
"no_record": 0,
|
||||
"recorded": 43,
|
||||
"unread": 0
|
||||
},
|
||||
"dev-php": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 3,
|
||||
"labels": 3,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 0,
|
||||
"recorded": 3,
|
||||
"unread": 0
|
||||
},
|
||||
"pairs": {
|
||||
"absent_fixed_side": 7,
|
||||
"examples": 740,
|
||||
"labels": 1544,
|
||||
"no_excerpt": 7,
|
||||
"no_record": 790,
|
||||
"recorded": 754,
|
||||
"unread": 0
|
||||
},
|
||||
"population-v1": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 85,
|
||||
"labels": 170,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 85,
|
||||
"recorded": 85,
|
||||
"unread": 0
|
||||
},
|
||||
"population-v2": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 42,
|
||||
"labels": 86,
|
||||
"no_excerpt": 1,
|
||||
"no_record": 43,
|
||||
"recorded": 43,
|
||||
"unread": 0
|
||||
}
|
||||
},
|
||||
"examples": 904,
|
||||
"examples_added": 180,
|
||||
"examples_before": 724,
|
||||
"failing_reason": "adjudications: 9 of 43 anonymous functions, unchanged since 2026-10-01 (exit 2 recorded, rows kept)",
|
||||
"failing_sources": [
|
||||
"adjudications"
|
||||
],
|
||||
"instrument_fix": "SideReader read only the first pair-kind source, so all 194 advisory sides were `unread` (0 examples) on the first rebuild; fixed in learn/examples.py with a test; the rebuild then read every side",
|
||||
"profile": "static"
|
||||
},
|
||||
"note": "Decision-engine harvest of the 97 advisory-fix pairs (benchmarks/pairs/advisory-fixes, 194 sides), COUNTS ONLY. Plane Runs hadv-verify (verify a/b + agree; verify families only: untrusted_destination) and hadv-roles (model roles, every side) on ax, generated by `ousast plane harvest --catalog benchmarks/pairs/advisory-fixes/catalog.toml --ceiling 2` from the label rows of those pairs; host engine features (benchmarks/learn/engine_pairs.py, frozen source, one container, --memory 3g); feature records (benchmarks/learn/build_features.py); static memory rebuilt with --units (old + new harvest units merged). Unit lists, label rows and excerpts name functions and stay local.",
|
||||
"roles": {
|
||||
"files_classified": 194,
|
||||
"role_rows": 660,
|
||||
"sides": 194,
|
||||
"skipped_units": {
|
||||
"oversized_over_20KB_inline": 0
|
||||
},
|
||||
"tasks_done": 20
|
||||
},
|
||||
"runner_image": "localhost:5001/ousast-runner@sha256:412db0a1504af8f1b23a395af8d1dcbd152c1bb6d16594f2b733067163a31884 (not rebuilt: no change under plane/tasks since it was built)",
|
||||
"spend": {
|
||||
"ceiling_usd": 5.0,
|
||||
"deepseek_balance_usd": {
|
||||
"after_features": 17.73,
|
||||
"after_runs": 17.75,
|
||||
"before": 17.88
|
||||
},
|
||||
"embeddings_openrouter_usd": 0.000868,
|
||||
"metered_usd": {
|
||||
"pilot_included": true,
|
||||
"roles": 0.1285,
|
||||
"total": 0.3802,
|
||||
"verify": 0.2517
|
||||
},
|
||||
"smoke": "plain JSON + one tool call through verify.build_client before any Run (one reply carried a stray `<|OPENAI|>` token in JSON mode on a trivial prompt; 0/16 on a classifier-shaped probe)",
|
||||
"task_budget_sum_usd": {
|
||||
"roles": 0.669,
|
||||
"verify": 0.816
|
||||
}
|
||||
},
|
||||
"verify": {
|
||||
"flagged_by_pass_units": {
|
||||
"fixed:none": 16,
|
||||
"fixed:one": 1,
|
||||
"vulnerable:both": 10,
|
||||
"vulnerable:none": 4,
|
||||
"vulnerable:one": 3
|
||||
},
|
||||
"hunts_per_pass": 34,
|
||||
"skipped_units": {
|
||||
"oversized_over_20KB_inline": 0
|
||||
},
|
||||
"tasks_done": 102,
|
||||
"units": 34,
|
||||
"usage": {
|
||||
"completion_tokens": 66734,
|
||||
"prompt_cache_hit_tokens": 444926,
|
||||
"prompt_tokens": 802586
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -265,8 +265,10 @@ class SideReader:
|
||||
if self._pairs is None:
|
||||
from ..pairs import load_pair_catalog
|
||||
|
||||
source = next(s for s in self.sources.sources if s.kind == "pairs")
|
||||
self._pairs = {case.name: case for case in load_pair_catalog(self.root / source.files[0])}
|
||||
# every pair-kind source (the corpus catalog and e.g. advisory-fixes): labels name both as source "pairs"
|
||||
self._pairs = {}
|
||||
for source in (s for s in self.sources.sources if s.kind == "pairs"):
|
||||
self._pairs.update({case.name: case for case in load_pair_catalog(self.root / source.files[0])})
|
||||
return self._pairs
|
||||
|
||||
def license(self, label: Mapping[str, Any]) -> str:
|
||||
|
||||
@@ -171,3 +171,34 @@ def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> Non
|
||||
assert build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES).summary()["examples"] == 0
|
||||
build = build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES, pins=pins)
|
||||
assert build.summary()["examples"] == 1 and build.rows[0]["pin"] == unit_pin
|
||||
|
||||
|
||||
def test_side_reader_reads_every_pair_kind_source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Labels from the corpus catalog and from a second pair-kind source (advisory-fixes) are both source ``pairs``;
|
||||
the reader must find either side, not only the first catalog's (the 2026-10-02 advisory rebuild read 0 of 194)."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
import openultrasast.pairs as pairs
|
||||
from openultrasast.learn.examples import SideReader
|
||||
|
||||
files = {}
|
||||
for name in ("corpus-case", "advisory-case"):
|
||||
files[name] = tmp_path / f"{name}.py"
|
||||
files[name].write_text("def handler(request):\n return 1\n")
|
||||
catalogs = {"corpus.toml": "corpus-case", "advisory.toml": "advisory-case"}
|
||||
|
||||
def fake(path: Path) -> tuple[Any, ...]:
|
||||
name = catalogs[Path(path).name]
|
||||
return (SimpleNamespace(name=name, vuln_file=files[name], fixed_file=files[name], license="MIT"),)
|
||||
|
||||
monkeypatch.setattr(pairs, "load_pair_catalog", fake)
|
||||
sources = Sources(
|
||||
(Source("pairs", "pairs", ("corpus.toml",), (), "2026-09-30", "development"),
|
||||
Source("advisory-fixes", "pairs", ("advisory.toml",), (), "2026-10-01", "development")),
|
||||
frozenset(), {}, Path("sources.toml"),
|
||||
) # fmt: skip
|
||||
reader = SideReader(tmp_path, tmp_path, sources, [])
|
||||
for name in catalogs.values():
|
||||
lab = label(source="pairs", source_ref=name, pin="")
|
||||
assert reader(lab, "vulnerable") == ["def handler(request):", " return 1"]
|
||||
assert reader.license(lab) == "MIT"
|
||||
|
||||
Reference in New Issue
Block a user