mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(learning-harness): the learning subcommands
Task 3.9. Five entry points make the classified detector set reachable: classify a corpus and report the classifier against itself, score a catalog by family, run round zero, run one evolve round, and publish every number from its artifacts. The pairs command gains a runs-per-pair flag, since the family scorer needs more than one. Every step degrades with a reason rather than failing: with no chat endpoint the run says so and still records what it could measure without a detector, and a missing configuration directory is written from the current hunter prompt rather than being an error. Artifacts are keyed by model throughout, so comparing a second detector is a flag rather than a rewrite. Both matrices are green and all three gate outputs stay byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
b8ad42c57e
commit
7a33756a7e
+154
-1
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
@@ -42,6 +42,7 @@ from .hunter import run_hunter_pool, write_hunter_trajectories
|
||||
from .hunter_harness import HxScanOrchestrator
|
||||
from .improve import RoundOutcome, run_improvement
|
||||
from .index import build_code_chunks
|
||||
from .learning.families import FamilyTaxonomy
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
||||
from .pair_gate import print_pair_metrics
|
||||
from .pairs import (
|
||||
@@ -184,6 +185,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
pairs.add_argument("--loo-out", type=Path, default=None, help="where to write loo.json (default: reports/loo.json)")
|
||||
pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON")
|
||||
pairs.add_argument("--k-runs", type=int, default=1, help="runs per pair on the hunter path; the family scorer needs at least three")
|
||||
|
||||
subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
|
||||
mechanisms = subparsers.add_parser("mechanisms", help="maintainer: mechanism memory seeded from trusted pairs")
|
||||
@@ -199,6 +201,32 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
export.add_argument("--json", action="store_true")
|
||||
|
||||
learning = subparsers.add_parser("learning", help="maintainer: the classified detector set and its rounds")
|
||||
learning_sub = learning.add_subparsers(dest="learning_command", required=True)
|
||||
for name, help_text in (
|
||||
("classify", "classify every labeled pair and report the classifier against itself"),
|
||||
("score", "score a catalog by family, with the denominator every number was computed over"),
|
||||
("baseline", "round zero: clone one detector into every family and measure each against itself"),
|
||||
("round", "one evolve round: propose one change for one family and let the evidence decide"),
|
||||
("publish", "regenerate every published number from the artifacts that produced it"),
|
||||
):
|
||||
command = learning_sub.add_parser(name, help=help_text)
|
||||
command.add_argument("--out", type=Path, default=Path(".openultrasast/learning"), help="where artifacts are read and written")
|
||||
if name != "publish":
|
||||
command.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
command.add_argument("--slice", choices=SLICE_NAMES, default="all")
|
||||
command.add_argument("--json", action="store_true")
|
||||
if name in {"baseline", "round", "score"}:
|
||||
command.add_argument("--k-runs", type=int, default=None, help="runs per pair; never fewer than three")
|
||||
command.add_argument("--model", default=None, help="detector model; artifacts are keyed by it")
|
||||
command.add_argument("--config", type=Path, default=Path("openultrasast.toml"))
|
||||
if name == "round":
|
||||
command.add_argument("--family", required=True)
|
||||
command.add_argument("--cost-cap-usd", type=float, default=None)
|
||||
if name == "publish":
|
||||
command.add_argument("--measurements", type=Path, default=Path("benchmarks/measurements"))
|
||||
command.add_argument("--roadmap", type=Path, default=Path(".kiro/steering/roadmap.md"))
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == "scan":
|
||||
return _scan(args.path, args.config, args.mode, args.fail_on)
|
||||
@@ -238,6 +266,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
if args.command == "mechanisms":
|
||||
return _mechanisms_export(args.catalog, args.slice, args.store, json_out=args.json)
|
||||
if args.command == "learning":
|
||||
return _learning(args)
|
||||
if args.command == "mcp":
|
||||
from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional
|
||||
|
||||
@@ -1162,6 +1192,129 @@ def _run_family_detectors(
|
||||
return findings, payload, cited
|
||||
|
||||
|
||||
def _learning(args: argparse.Namespace) -> int:
|
||||
"""The maintainer's entry points into the classified detector set. Everything degrades with a reason."""
|
||||
from .learning.classify import measure_classifier, write_review_queue
|
||||
from .learning.families import load_families
|
||||
from .learning.publish import publish
|
||||
|
||||
out: Path = args.out
|
||||
taxonomy = load_families()
|
||||
if args.learning_command == "publish":
|
||||
report = publish(learning_dir=out, measurements_dir=args.measurements, roadmap=args.roadmap)
|
||||
print(f"learning publish: {len(report.families)} families over {len(report.models)} model(s) -> {args.roadmap}")
|
||||
for artifact in report.artifacts:
|
||||
print(f" artifact {artifact}")
|
||||
return 0
|
||||
cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice))
|
||||
if args.learning_command == "classify":
|
||||
measured = measure_classifier(cases, taxonomy, client=None)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
payload = measured.to_dict()
|
||||
(out / "classifier.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
written = write_review_queue(out / "review-queue.jsonl", measured.queue)
|
||||
if args.json:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"learning classify: classified {measured.classified}, unknown {measured.unknown}, queued {written}")
|
||||
return 0
|
||||
return _learning_run(args, cases, taxonomy)
|
||||
|
||||
|
||||
def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: FamilyTaxonomy) -> int:
|
||||
"""score, baseline and round: the three that need a detector, and degrade with a reason without one."""
|
||||
from .learning.detectors import load_family_configs, write_default_configs
|
||||
from .learning.endpoint import resolve_chat_endpoint, resolve_models
|
||||
from .learning.journal import Archive, LearningJournal
|
||||
from .learning.proposer import HarnessXProposer
|
||||
from .learning.rounds import MIN_K_RUNS, load_noise_floors, run_baseline, run_learning_round
|
||||
from .learning.scoring import aggregate
|
||||
from .tool_hunter import _SYSTEM_PROMPT
|
||||
|
||||
out: Path = args.out
|
||||
config = load_config(args.config if args.config.exists() else None)
|
||||
k_runs = args.k_runs or max(config.learning.k_runs, MIN_K_RUNS)
|
||||
model = args.model or resolve_models(config)[0]
|
||||
configs_dir = out / "configs"
|
||||
if not configs_dir.exists():
|
||||
write_default_configs(configs_dir, taxonomy, prompt=_SYSTEM_PROMPT, version="0") # type: ignore[arg-type]
|
||||
resolved = resolve_chat_endpoint(config)
|
||||
if resolved is None:
|
||||
print("learning: learning_endpoint_unavailable; scoring what can be scored without a detector")
|
||||
client = resolved[0] if resolved else None
|
||||
configs = load_family_configs(configs_dir, taxonomy) # type: ignore[arg-type]
|
||||
|
||||
def factory(family_config: object) -> Callable[[Path], list[StaticFinding]]:
|
||||
from .learning.detectors import Region, run_family_detector
|
||||
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if client is None:
|
||||
return []
|
||||
region = Region(path=_first_source(root), function=None)
|
||||
return run_family_detector(root, region, family_config, client=client, model=model) # type: ignore[arg-type]
|
||||
|
||||
return scan
|
||||
|
||||
if args.learning_command == "score":
|
||||
from .learning.rounds import score_case
|
||||
|
||||
scores = [
|
||||
score_case(case, factory(configs[_label(case)]), taxonomy=taxonomy, runs=k_runs) for case in cases if _label(case) in configs
|
||||
]
|
||||
families = {name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy).items()}
|
||||
if args.json:
|
||||
print(json.dumps({"families": families}, indent=2, sort_keys=True))
|
||||
else:
|
||||
for name, block in sorted(families.items()):
|
||||
print(f"learning score {name}: scorable={block['scorable']} recall={block['recall']:.3f} youden={block['youden']:.3f}")
|
||||
return 0
|
||||
if args.learning_command == "baseline":
|
||||
report = run_baseline(
|
||||
cases,
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=configs_dir,
|
||||
scan=factory(configs.get("unknown") or next(iter(configs.values()))),
|
||||
model=model,
|
||||
out_dir=out,
|
||||
k_runs=k_runs,
|
||||
)
|
||||
print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}")
|
||||
return 0
|
||||
journal = LearningJournal(out / "journal.jsonl")
|
||||
record = run_learning_round(
|
||||
cases,
|
||||
family=args.family,
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=configs_dir,
|
||||
proposer=HarnessXProposer(configs_dir=configs_dir, model=model),
|
||||
scan_factory=factory,
|
||||
model=model,
|
||||
journal=journal,
|
||||
archive=Archive(out / "archive.jsonl"),
|
||||
floors=load_noise_floors(out, model),
|
||||
out_dir=out,
|
||||
cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd,
|
||||
minibatch=config.learning.minibatch,
|
||||
k_runs=k_runs,
|
||||
)
|
||||
print(f"learning round {record.round} {record.family}: {record.outcome} ({record.reason})")
|
||||
return 0
|
||||
|
||||
|
||||
def _label(case: PairCase) -> str:
|
||||
for row in case.expected:
|
||||
if row.family:
|
||||
return str(row.family)
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _first_source(root: Path) -> str:
|
||||
from .preprocess import preprocess_repository
|
||||
|
||||
_, targets = preprocess_repository(root)
|
||||
return targets[0].path if targets else "."
|
||||
|
||||
|
||||
def _hotspot_from_variant(finding: StaticFinding) -> Hotspot:
|
||||
return hotspot_from_finding(finding, rationale="variant of a known mechanism (suspicion; sandbox may raise)")
|
||||
|
||||
|
||||
@@ -241,7 +241,7 @@ def run_learning_round(
|
||||
sweep_families: Sequence[str] = (),
|
||||
spent_usd: Callable[[], float] | None = None,
|
||||
trajectories: Callable[[], Sequence[Mapping[str, object]]] | None = None,
|
||||
) -> object:
|
||||
) -> RoundRecord:
|
||||
"""One round: propose one change for one family, then let the evidence decide whether it stays.
|
||||
|
||||
Staged so that the cheap answer comes first: a train minibatch, then the family's held-out pairs,
|
||||
@@ -437,7 +437,7 @@ def _finish(
|
||||
scores: Sequence[PairFamilyScore] = (),
|
||||
version: str | None = None,
|
||||
trajectories: Callable[[], Sequence[Mapping[str, object]]] | None = None,
|
||||
) -> object:
|
||||
) -> RoundRecord:
|
||||
"""Write the round directory and journal the outcome. Every outcome is journalled, not just the good ones."""
|
||||
from ..redaction import redact_secrets
|
||||
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""learning-harness task 3.9: the learning subcommands (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n"
|
||||
CATALOG = (
|
||||
'[[pair]]\nname = "{name}"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "{name}-v.py"\nfixed = "{name}-f.py"\n'
|
||||
'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "seeded"\n\n'
|
||||
'[[pair.expected]]\ncwe = "CWE-78"\nclass = "command injection"\npath = "app.py"\nfunction = "run"\n'
|
||||
'sink = "system"\nmechanism = "source_reaches_sink"\nfamily = "injection"\n'
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
for name in ("train-a", "hold-a"):
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
(tmp_path / "pairs.toml").write_text(
|
||||
CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout")
|
||||
)
|
||||
monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted")
|
||||
return tmp_path # no chdir: the mechanism vocabulary is found relative to the repository root
|
||||
|
||||
|
||||
def test_classify_reports_the_corpus_and_writes_the_queue(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert main(["learning", "classify", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning")]) == 0
|
||||
payload = json.loads((workspace / "learning" / "classifier.json").read_text())
|
||||
assert payload["classified"] == 2 and payload["per_family"]["injection"] == 2
|
||||
assert payload["hierarchical_credit"] is False
|
||||
assert "classified 2" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_score_reports_family_numbers_for_a_catalog(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert main(["learning", "score", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--json"]) == 0
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
assert "injection" in payload["families"]
|
||||
assert payload["families"]["injection"]["taxonomy_version"]
|
||||
|
||||
|
||||
def test_baseline_writes_the_noise_floor_and_publish_reads_it(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"scripted",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
floors = json.loads((workspace / "learning" / "baseline" / "scripted" / "noise-floors.json").read_text())
|
||||
assert floors["k_runs"] == 3 and "injection" in floors["floors"]
|
||||
capsys.readouterr()
|
||||
roadmap = workspace / "roadmap.md"
|
||||
roadmap.write_text("# Roadmap\n\n## Overview\n\ntext\n")
|
||||
assert (
|
||||
main(
|
||||
["learning", "publish", "--out", str(workspace / "learning"), "--measurements", str(workspace / "m"), "--roadmap", str(roadmap)]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert "learning-harness:begin" in roadmap.read_text()
|
||||
assert "| injection |" in roadmap.read_text()
|
||||
|
||||
|
||||
def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace: Path) -> None:
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"scripted",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"round",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--family",
|
||||
"injection",
|
||||
"--k-runs",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
rounds = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()]
|
||||
assert len(rounds) == 1 and rounds[0]["family"] == "injection"
|
||||
assert rounds[0]["outcome"] in {"accepted", "rejected", "reverted_cost"}
|
||||
assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file()
|
||||
|
||||
|
||||
def test_a_missing_endpoint_is_recorded_and_the_rest_still_runs(workspace: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"none",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert "learning_endpoint_unavailable" in out
|
||||
assert (workspace / "learning" / "baseline" / "none" / "report.json").is_file() # the run still recorded what it could
|
||||
|
||||
|
||||
def test_every_subcommand_is_listed_in_help(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
main(["learning", "--help"])
|
||||
out = capsys.readouterr().out
|
||||
for name in ("classify", "score", "baseline", "round", "publish"):
|
||||
assert name in out
|
||||
|
||||
|
||||
def test_the_pairs_command_takes_a_k_runs_flag(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
main(["pairs", "--help"])
|
||||
assert "--k-runs" in capsys.readouterr().out
|
||||
Reference in New Issue
Block a user