mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
e44d45f488
The maintainer settled both items the group 1 review escalated. The family that covers server-side request forgery is named untrusted destination and also covers open redirect, because both are untrusted input choosing where a request goes and the corpus carries 34 open-redirect rows that would otherwise be unclassifiable. The requirement, the brief and the design example now say so, and the approval note records the amendment. The shipped taxonomy is flat, so no answer can earn hierarchical partial credit. Rather than leave that as prose, two new acceptance criteria require every agreement figure and every family figure to say it, and both report types now carry the flag, computed from the taxonomy rather than asserted. A taxonomy that later declares a parent flips it without any other change. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
112 lines
5.4 KiB
Python
112 lines
5.4 KiB
Python
"""learning-harness task 2.2: the classifier reports on itself and queues disagreements (offline)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.learning.families import load_families
|
|
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored
|
|
|
|
|
|
def _case(tmp_path: Path, name: str, *, cwe: str, mechanism: str = "other", family: str | None = None, tier: str = "reviewed") -> PairCase:
|
|
(tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
|
|
return PairCase(
|
|
name=name,
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / f"{name}.py",
|
|
fixed_file=tmp_path / f"{name}.py",
|
|
relpath="app.py",
|
|
expected=(
|
|
ExpectedFinding(cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function="f", mechanism=mechanism, family=family),
|
|
),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
review_tier=tier,
|
|
reviewer="m" if tier == "reviewed" else "",
|
|
)
|
|
|
|
|
|
def test_the_report_covers_the_whole_corpus_and_names_the_unknowns(tmp_path: Path) -> None:
|
|
from openultrasast.learning.classify import measure_classifier
|
|
|
|
taxonomy = load_families()
|
|
cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG))
|
|
report = measure_classifier(cases, taxonomy, client=None)
|
|
assert report.taxonomy_version == taxonomy.version
|
|
assert report.classified == len(cases)
|
|
assert sum(report.per_tier.values()) == len(cases) and set(report.per_tier) <= {"static", "model", "none"}
|
|
assert report.per_family["memory"] >= 150 # the vfc rows land in one family instead of reading `other`
|
|
assert report.per_family["access_control"] >= 10 and report.per_family["injection"] >= 40
|
|
assert report.unknown == 2 and report.per_tier["static"] == len(cases) - 2 # tier one places all but two rows
|
|
payload = report.to_dict()
|
|
json.dumps(payload)
|
|
assert payload["labeled"] == 0 # no maintainer family labels in the corpus yet, and the report says so
|
|
|
|
|
|
def test_agreement_uses_the_first_answer_and_the_oracle_uses_any_of_them(tmp_path: Path) -> None:
|
|
from openultrasast.learning.classify import measure_classifier
|
|
|
|
taxonomy = load_families()
|
|
cases = [
|
|
_case(tmp_path, "hit", cwe="CWE-89", family="injection"), # tier one answers injection: agreement and oracle
|
|
_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), # answers injection: neither
|
|
]
|
|
report = measure_classifier(cases, taxonomy, client=None)
|
|
assert report.labeled == 2
|
|
assert report.agreement == 0.5 and report.oracle_router == 0.5
|
|
assert abs(report.random_router - 1 / 9) < 1e-9 # a uniform pick over the nine real families
|
|
assert report.confusion["injection"]["injection"] == 1 and report.confusion["access_control"]["injection"] == 1
|
|
|
|
|
|
def test_every_disagreement_is_queued_once_with_both_answers_and_the_tier(tmp_path: Path) -> None:
|
|
from openultrasast.learning.classify import measure_classifier, write_review_queue
|
|
|
|
taxonomy = load_families()
|
|
cases = [_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), _case(tmp_path, "hit", cwe="CWE-89", family="injection")]
|
|
report = measure_classifier(cases, taxonomy, client=None)
|
|
assert [entry.pair for entry in report.queue] == ["miss"]
|
|
entry = report.queue[0]
|
|
assert entry.label == "access_control" and entry.classified == ("injection",) and entry.tier == "static"
|
|
queue_path = tmp_path / "review-queue.jsonl"
|
|
write_review_queue(queue_path, report.queue)
|
|
write_review_queue(queue_path, report.queue) # a second measurement must not duplicate the row
|
|
rows = [json.loads(line) for line in queue_path.read_text().splitlines() if line.strip()]
|
|
assert [row["pair"] for row in rows] == ["miss"]
|
|
assert rows[0]["label"] == "access_control" and rows[0]["classified"] == ["injection"] and rows[0]["tier"] == "static"
|
|
|
|
|
|
def test_measuring_never_writes_a_label(tmp_path: Path) -> None:
|
|
from openultrasast.learning.classify import measure_classifier
|
|
|
|
taxonomy = load_families()
|
|
case = _case(tmp_path, "miss", cwe="CWE-89", family="access_control")
|
|
measure_classifier([case], taxonomy, client=None)
|
|
assert case.expected[0].family == "access_control" # the classifier proposes; the maintainer decides
|
|
|
|
|
|
def test_only_reviewed_and_seeded_rows_count_towards_agreement(tmp_path: Path) -> None:
|
|
from openultrasast.learning.classify import measure_classifier
|
|
|
|
taxonomy = load_families()
|
|
cases = [
|
|
_case(tmp_path, "trusted", cwe="CWE-89", family="access_control", tier="reviewed"),
|
|
_case(tmp_path, "untrusted", cwe="CWE-89", family="access_control", tier="title"),
|
|
]
|
|
report = measure_classifier(cases, taxonomy, client=None)
|
|
assert report.labeled == 1 and report.classified == 2 # an unreviewed label is not a reference answer
|
|
assert [entry.pair for entry in report.queue] == ["trusted"]
|
|
|
|
|
|
def test_the_report_says_when_hierarchical_credit_could_not_apply(tmp_path: Path) -> None:
|
|
"""Req 2.6: the shipped taxonomy is flat, so partial credit is inert and every figure must say so."""
|
|
from openultrasast.learning.classify import measure_classifier
|
|
|
|
taxonomy = load_families()
|
|
report = measure_classifier([_case(tmp_path, "a", cwe="CWE-89", family="injection")], taxonomy, client=None)
|
|
assert report.hierarchical_credit is False
|
|
assert report.to_dict()["hierarchical_credit"] is False
|