mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
1843ef9a02
Eight of nine group-3 tasks were rejected. Each defect was hiding a number, and the numbers are why the loop was not learning: - The fixed side was judged against the *vulnerable* side's function ranges. 191 of 227 pairs with a resolvable labeled span (84%) have different spans on the two sides, so a leak was recorded as silence, always in the direction that flatters Youden. - Round zero measured every family with the generalist. `family:unknown` earns no credit against any real family, so every family but `unknown` baselined at a structural zero and every noise floor with it. - The cross-family sweep counted absolute failures instead of negative flips. On a corpus where most pairs fail at round zero that rejects every round for damage it did not do. Unscorable rows voted too. - K runs was declared and never threaded; `pairs --k-runs` was parsed and discarded, and its only test grepped `--help`. - The meta-agent proposer was a stub, so `learning round` could never propose anything. It now composes MetaAgent.evolve against a scratch copy of the family directory, and `--proposals FILE` runs a recorded sequence offline. - `refused_holdout` was a vocabulary word no code path could produce, and the covering test accepted either outcome. - Cost was metered once, after everything had been paid for; a stage that raised left no record at all. - The corrected lever number was published beside the earlier one rather than in place of it, and the test named for that property never checked. - Refusals were `continue` statements: `Refusal.degradation()` existed and nothing called it. A train-split identical twin was reported as `split:train`, which is untrue. Test repairs that each changed a number: two split tests measured an `x = 1` fixture that teaches nothing whatever the rule says; two CLI scoreboard tests asserted against all-zero output produced by a stub that answers twice and puts its finding outside the labeled function. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
95 lines
4.3 KiB
Python
95 lines
4.3 KiB
Python
"""A refusal is data, not a `continue` (learning-harness review round 3, Req 5.1, 5.2).
|
|
|
|
`Refusal.degradation()` existed and nothing ever called it, so every path that declined to learn from a pair
|
|
declined silently: the mechanism lever skipped a holdout-taught shape without a word, leave-one-out dropped
|
|
non-teachers with no record, and the seeder gave a train-split identical twin the reason `split:train`, which
|
|
is not true. A leak the run does not mention is a leak nobody reads.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.pairs import PairCase
|
|
|
|
VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
|
FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n"
|
|
|
|
|
|
def _case(tmp_path: Path, name: str, *, split: str = "train", unscorable: str | None = None, tier: str = "seeded") -> PairCase:
|
|
(tmp_path / f"{name}-v.py").write_text(VULN)
|
|
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
|
return PairCase(
|
|
name=name,
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / f"{name}-v.py",
|
|
fixed_file=tmp_path / f"{name}-f.py",
|
|
relpath="app.py",
|
|
expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
split=split,
|
|
review_tier=tier,
|
|
unscorable=unscorable,
|
|
)
|
|
|
|
|
|
def test_a_train_split_twin_is_refused_for_being_a_twin_not_for_its_split(tmp_path: Path) -> None:
|
|
"""`split:train` on a train-split row is a lie, and it hides a corpus fact behind a split fact.
|
|
|
|
Two vibe-py rows are exactly this: `python-insecure-app-main-try-hack-me` and
|
|
`vulnpy-deserialization-do-pickle-load` are train-split identical twins."""
|
|
from openultrasast.semantic.seed import _skip_reason
|
|
|
|
assert _skip_reason(_case(tmp_path, "twin", split="train", unscorable="identical_twin")) == "unscorable:identical_twin"
|
|
assert _skip_reason(_case(tmp_path, "held", split="holdout")) == "split:holdout"
|
|
assert _skip_reason(_case(tmp_path, "good", split="train")) is None
|
|
|
|
|
|
def test_leave_one_out_records_the_pairs_it_would_not_learn_from(tmp_path: Path) -> None:
|
|
"""Req 5.2: the refusal is recorded with the pair names, on a channel a caller can read."""
|
|
from openultrasast.semantic.loo import evaluate_loo
|
|
|
|
result = evaluate_loo([_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")])
|
|
refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"]
|
|
assert refusals, "leave-one-out excluded a holdout pair from teaching and said nothing"
|
|
assert refusals[0]["pairs"] == ["hold-a"]
|
|
assert result.teaching_pairs <= 1
|
|
|
|
|
|
def test_the_mechanism_lever_records_every_shape_a_holdout_pair_taught(tmp_path: Path) -> None:
|
|
"""The lever used to `continue` past a refused candidate; the run then showed a shorter list with no reason."""
|
|
from openultrasast.improve.evolve import propose_mechanism_edits
|
|
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
|
from openultrasast.semantic.variants import Shape
|
|
|
|
cases = [_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")]
|
|
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
|
append_from_pair(
|
|
candidates,
|
|
Shape(
|
|
language="python",
|
|
sink_name="system",
|
|
arity=1,
|
|
source_positions=(0,),
|
|
source_kinds=("request",),
|
|
guard="none",
|
|
mechanism="source_reaches_sink",
|
|
),
|
|
summary="s",
|
|
cwe="CWE-78",
|
|
pair="hold-a",
|
|
provenance="human",
|
|
tier="seeded",
|
|
)
|
|
refusals: list[dict[str, object]] = []
|
|
edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl", degradations=refusals)
|
|
assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == []
|
|
assert [item["reason"] for item in refusals] == ["holdout_pair_refused"]
|
|
assert refusals[0]["pairs"] == ["hold-a"]
|
|
json.dumps(refusals) # a degradation has to survive the artifact it is written into
|